From 41b9d1e3b087de40b3b6454d1ba263eec26a58c5 Mon Sep 17 00:00:00 2001 From: Derek Keeler Date: Tue, 25 Aug 2026 20:04:14 -0700 Subject: [PATCH 1/3] Harden GC log decompression limits Bound compressed and expanded input, line length, archive structure, and rotating log processing. Preserve limit and archive integrity failures for callers. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../com/microsoft/gctoolkit/GCToolKit.java | 6 + .../gctoolkit/io/ArchiveSliceInputStream.java | 65 ++ .../gctoolkit/io/BoundedLineSpliterator.java | 135 +++ .../gctoolkit/io/CountingInputStream.java | 78 ++ .../microsoft/gctoolkit/io/DataSource.java | 2 + .../com/microsoft/gctoolkit/io/GCLogFile.java | 35 +- .../gctoolkit/io/GCLogFileSegment.java | 125 +-- .../gctoolkit/io/GCLogFileZipSegment.java | 91 +- .../gctoolkit/io/LimitedGZIPInputStream.java | 279 ++++++ .../gctoolkit/io/LimitedInputStream.java | 95 ++ .../gctoolkit/io/LogFileReadBudget.java | 96 +++ .../io/LogFileReadLimitExceededException.java | 103 +++ .../gctoolkit/io/LogFileReadLimits.java | 241 ++++++ .../gctoolkit/io/LogFileStreams.java | 180 ++++ .../gctoolkit/io/RotatingGCLogFile.java | 95 +- .../gctoolkit/io/RotatingLogFileMetadata.java | 135 ++- .../gctoolkit/io/SingleGCLogFile.java | 55 +- .../gctoolkit/io/SingleLogFileMetadata.java | 6 +- .../io/ValidatedZipEntryInputStream.java | 277 ++++++ .../gctoolkit/io/ZipArchivePreflight.java | 490 +++++++++++ .../jvm/AbstractJavaVirtualMachine.java | 8 +- .../io/SingleGCLogFileReadLimitsTest.java | 812 ++++++++++++++++++ 22 files changed, 3164 insertions(+), 245 deletions(-) create mode 100644 api/src/main/java/com/microsoft/gctoolkit/io/ArchiveSliceInputStream.java create mode 100644 api/src/main/java/com/microsoft/gctoolkit/io/BoundedLineSpliterator.java create mode 100644 api/src/main/java/com/microsoft/gctoolkit/io/CountingInputStream.java create mode 100644 api/src/main/java/com/microsoft/gctoolkit/io/LimitedGZIPInputStream.java create mode 100644 api/src/main/java/com/microsoft/gctoolkit/io/LimitedInputStream.java create mode 100644 api/src/main/java/com/microsoft/gctoolkit/io/LogFileReadBudget.java create mode 100644 api/src/main/java/com/microsoft/gctoolkit/io/LogFileReadLimitExceededException.java create mode 100644 api/src/main/java/com/microsoft/gctoolkit/io/LogFileReadLimits.java create mode 100644 api/src/main/java/com/microsoft/gctoolkit/io/LogFileStreams.java create mode 100644 api/src/main/java/com/microsoft/gctoolkit/io/ValidatedZipEntryInputStream.java create mode 100644 api/src/main/java/com/microsoft/gctoolkit/io/ZipArchivePreflight.java create mode 100644 api/src/test/java/com/microsoft/gctoolkit/io/SingleGCLogFileReadLimitsTest.java diff --git a/api/src/main/java/com/microsoft/gctoolkit/GCToolKit.java b/api/src/main/java/com/microsoft/gctoolkit/GCToolKit.java index 6c258dcb7..513e1e33b 100644 --- a/api/src/main/java/com/microsoft/gctoolkit/GCToolKit.java +++ b/api/src/main/java/com/microsoft/gctoolkit/GCToolKit.java @@ -7,6 +7,7 @@ import com.microsoft.gctoolkit.aggregator.EventSource; import com.microsoft.gctoolkit.io.DataSource; import com.microsoft.gctoolkit.io.GCLogFile; +import com.microsoft.gctoolkit.io.LogFileReadLimitExceededException; import com.microsoft.gctoolkit.io.RotatingGCLogFile; import com.microsoft.gctoolkit.io.SingleGCLogFile; import com.microsoft.gctoolkit.jvm.Diary; @@ -16,6 +17,7 @@ import com.microsoft.gctoolkit.message.JVMEventChannel; import java.io.IOException; +import java.io.UncheckedIOException; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Parameter; @@ -312,6 +314,10 @@ public JavaVirtualMachine analyze(DataSource dataSource) throws IOException long start = System.currentTimeMillis(); javaVirtualMachine.analyze(filteredAggregators, jvmEventChannel, dataSourceChannel); LOGGER.log(Level.FINE,() -> "Analysis completed in " + (System.currentTimeMillis() - start) + "ms"); + } catch (LogFileReadLimitExceededException limitExceeded) { + throw limitExceeded; + } catch (UncheckedIOException uncheckedIOException) { + throw uncheckedIOException.getCause(); } catch(Throwable t) { LOGGER.log(Level.SEVERE, "Internal Error: Cannot invoke analyze method", t); } diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/ArchiveSliceInputStream.java b/api/src/main/java/com/microsoft/gctoolkit/io/ArchiveSliceInputStream.java new file mode 100644 index 000000000..1f9deeb6e --- /dev/null +++ b/api/src/main/java/com/microsoft/gctoolkit/io/ArchiveSliceInputStream.java @@ -0,0 +1,65 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +package com.microsoft.gctoolkit.io; + +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; + +final class ArchiveSliceInputStream extends InputStream { + + private final FileInputStream inputStream; + private long remaining; + + ArchiveSliceInputStream(Path path, long offset, long length) throws IOException { + inputStream = new FileInputStream(path.toFile()); + try { + inputStream.getChannel().position(offset); + remaining = length; + } catch (IOException | RuntimeException | Error failure) { + try { + inputStream.close(); + } catch (IOException closeException) { + failure.addSuppressed(closeException); + } + throw failure; + } + } + + @Override + public int read() throws IOException { + if (remaining == 0L) { + return -1; + } + int value = inputStream.read(); + if (value != -1) { + remaining--; + } + return value; + } + + @Override + public int read(byte[] bytes, int offset, int length) throws IOException { + if (remaining == 0L) { + return -1; + } + int count = inputStream.read(bytes, offset, (int) Math.min(length, remaining)); + if (count > 0) { + remaining -= count; + } + return count; + } + + @Override + public long skip(long count) throws IOException { + long skipped = inputStream.skip(Math.min(count, remaining)); + remaining -= skipped; + return skipped; + } + + @Override + public void close() throws IOException { + inputStream.close(); + } +} diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/BoundedLineSpliterator.java b/api/src/main/java/com/microsoft/gctoolkit/io/BoundedLineSpliterator.java new file mode 100644 index 000000000..e2220a5e4 --- /dev/null +++ b/api/src/main/java/com/microsoft/gctoolkit/io/BoundedLineSpliterator.java @@ -0,0 +1,135 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +package com.microsoft.gctoolkit.io; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.Reader; +import java.io.UncheckedIOException; +import java.nio.file.Path; +import java.util.Objects; +import java.util.Spliterator; +import java.util.function.Consumer; + +final class BoundedLineSpliterator implements Spliterator { + + private static final int NO_PENDING_CHARACTER = -2; + + private final BufferedReader reader; + private final int maximumLineCharacters; + private final Path path; + private final String archiveEntry; + private int pendingCharacter = NO_PENDING_CHARACTER; + private boolean closed; + + BoundedLineSpliterator( + Reader reader, + int maximumLineCharacters, + Path path, + String archiveEntry) { + this.reader = new BufferedReader(reader); + this.maximumLineCharacters = maximumLineCharacters; + this.path = path; + this.archiveEntry = archiveEntry; + } + + @Override + public boolean tryAdvance(Consumer action) { + Objects.requireNonNull(action); + if (closed) { + return false; + } + + StringBuilder line = new StringBuilder(Math.min(maximumLineCharacters, 8192)); + try { + while (true) { + int character = readCharacter(); + if (character == -1) { + close(); + if (line.length() == 0) { + return false; + } + action.accept(line.toString()); + return true; + } + if (character == '\n') { + action.accept(line.toString()); + return true; + } + if (character == '\r') { + int following = readCharacter(); + if (following != '\n' && following != -1) { + pendingCharacter = following; + } + action.accept(line.toString()); + if (following == -1) { + close(); + } + return true; + } + if (line.length() == maximumLineCharacters) { + throw new LogFileReadLimitExceededException( + LogFileReadLimitExceededException.LimitType.LINE_CHARACTERS, + path, + archiveEntry, + Integer.toString(maximumLineCharacters), + Integer.toString(maximumLineCharacters + 1)); + } + line.append((char) character); + } + } catch (IOException exception) { + closeAfterFailure(exception); + throw new UncheckedIOException(exception); + } catch (RuntimeException exception) { + closeAfterFailure(exception); + throw exception; + } + } + + private int readCharacter() throws IOException { + if (pendingCharacter != NO_PENDING_CHARACTER) { + int character = pendingCharacter; + pendingCharacter = NO_PENDING_CHARACTER; + return character; + } + return reader.read(); + } + + private void closeAfterFailure(Throwable failure) { + try { + close(); + } catch (IOException closeException) { + failure.addSuppressed(closeException); + } + } + + void closeUnchecked() { + try { + close(); + } catch (IOException exception) { + throw new UncheckedIOException(exception); + } + } + + private void close() throws IOException { + if (!closed) { + closed = true; + reader.close(); + } + } + + @Override + public Spliterator trySplit() { + return null; + } + + @Override + public long estimateSize() { + return Long.MAX_VALUE; + } + + @Override + public int characteristics() { + return ORDERED | NONNULL; + } +} diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/CountingInputStream.java b/api/src/main/java/com/microsoft/gctoolkit/io/CountingInputStream.java new file mode 100644 index 000000000..de6feaf55 --- /dev/null +++ b/api/src/main/java/com/microsoft/gctoolkit/io/CountingInputStream.java @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +package com.microsoft.gctoolkit.io; + +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; + +final class CountingInputStream extends FilterInputStream { + + private final long maximumBytes; + private final Path path; + private long bytesRead; + + CountingInputStream(InputStream inputStream, long maximumBytes, Path path) { + super(inputStream); + this.maximumBytes = maximumBytes; + this.path = path; + } + + long getBytesRead() { + return bytesRead; + } + + @Override + public int read() throws IOException { + int value = super.read(); + if (value != -1) { + record(1L); + } + return value; + } + + @Override + public int read(byte[] bytes, int offset, int length) throws IOException { + int count = super.read(bytes, offset, nextReadLength(length)); + if (count > 0) { + record(count); + } + return count; + } + + @Override + public long skip(long count) throws IOException { + long skipped = super.skip(nextReadLength(count)); + record(skipped); + return skipped; + } + + private int nextReadLength(int requestedLength) { + return (int) nextReadLength((long) requestedLength); + } + + private long nextReadLength(long requestedLength) { + long remaining = maximumBytes - bytesRead; + long detectableLength = remaining == Long.MAX_VALUE ? Long.MAX_VALUE : remaining + 1L; + return Math.min(requestedLength, detectableLength); + } + + private void record(long count) { + if (count <= 0L) { + return; + } + if (count > maximumBytes - bytesRead) { + long observed = bytesRead == Long.MAX_VALUE + ? Long.MAX_VALUE + : bytesRead + count; + throw new LogFileReadLimitExceededException( + LogFileReadLimitExceededException.LimitType.COMPRESSED_BYTES, + path, + null, + Long.toString(maximumBytes), + Long.toString(observed)); + } + bytesRead += count; + } +} diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/DataSource.java b/api/src/main/java/com/microsoft/gctoolkit/io/DataSource.java index de193bc91..0ec11ce5b 100644 --- a/api/src/main/java/com/microsoft/gctoolkit/io/DataSource.java +++ b/api/src/main/java/com/microsoft/gctoolkit/io/DataSource.java @@ -25,6 +25,8 @@ public interface DataSource { * @return A stream of the data. * @throws IOException Thrown if the data cannot be streamed, * or an IOException is raised while streaming. + * @throws LogFileReadLimitExceededException if a log-file resource limit is exceeded during a + * lazy terminal operation */ Stream stream() throws IOException; diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/GCLogFile.java b/api/src/main/java/com/microsoft/gctoolkit/io/GCLogFile.java index cf0e7e636..513fc568b 100644 --- a/api/src/main/java/com/microsoft/gctoolkit/io/GCLogFile.java +++ b/api/src/main/java/com/microsoft/gctoolkit/io/GCLogFile.java @@ -40,13 +40,33 @@ public abstract class GCLogFile extends FileDataSource { private Diary diary; private TripleState unifiedFormat = TripleState.UNKNOWN; private JavaVirtualMachine jvm = null; + private final LogFileReadLimits readLimits; /** * Subclass only. * @param path The path to the GCLogFile or, in the case of rotating log files, the parent directory. */ protected GCLogFile(Path path) { + this(path, LogFileReadLimits.defaults()); + } + + /** + * Subclass only. + * @param path The path to the GCLogFile or, in the case of rotating log files, the parent directory. + * @param readLimits resource limits applied while streaming the log + */ + protected GCLogFile(Path path, LogFileReadLimits readLimits) { super(path); + this.readLimits = Objects.requireNonNull(readLimits, "readLimits"); + } + + /** + * Returns the resource limits applied while streaming this log. + * + * @return immutable read limits + */ + public final LogFileReadLimits getReadLimits() { + return readLimits; } /** @@ -96,13 +116,14 @@ private Diarizer diarizer() { public Diary diary() throws IOException { if ( diary == null) { Diarizer diarizer = diarizer(); - stream() - .filter(Objects::nonNull) - .map(String::trim) - .filter(s -> s.length() > 0) - .map(diarizer::diarize) - .filter(completed -> completed) - .findFirst(); + try (Stream lines = stream()) { + lines.filter(Objects::nonNull) + .map(String::trim) + .filter(s -> s.length() > 0) + .map(diarizer::diarize) + .filter(completed -> completed) + .findFirst(); + } this.diary = diarizer.getDiary(); } return diary; diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/GCLogFileSegment.java b/api/src/main/java/com/microsoft/gctoolkit/io/GCLogFileSegment.java index b52f6cbb7..9a00244b1 100644 --- a/api/src/main/java/com/microsoft/gctoolkit/io/GCLogFileSegment.java +++ b/api/src/main/java/com/microsoft/gctoolkit/io/GCLogFileSegment.java @@ -5,12 +5,16 @@ import com.microsoft.gctoolkit.time.DateTimeStamp; import java.io.IOException; -import java.io.RandomAccessFile; -import java.nio.file.Files; +import java.io.UncheckedIOException; import java.nio.file.Path; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Comparator; +import java.util.Deque; +import java.util.List; +import java.util.Objects; import java.util.regex.Matcher; +import java.util.stream.Collector; import java.util.stream.Stream; /** @@ -23,6 +27,7 @@ public class GCLogFileSegment implements LogFileSegment { private final Path path; + private final LogFileReadLimits readLimits; private final int segmentIndex; private final boolean current; private DateTimeStamp endTime = null; @@ -33,7 +38,18 @@ public class GCLogFileSegment implements LogFileSegment { * @param path The path to the file. */ public GCLogFileSegment(Path path) { + this(path, LogFileReadLimits.defaults()); + } + + /** + * Creates a log segment with explicit resource limits. + * + * @param path the path to the file + * @param readLimits resource limits applied while streaming the segment + */ + public GCLogFileSegment(Path path, LogFileReadLimits readLimits) { this.path = path; + this.readLimits = Objects.requireNonNull(readLimits, "readLimits"); String filename = path.getFileName().toString(); Matcher matcher = ROTATING_LOG_PATTERN.matcher(filename); @@ -70,8 +86,12 @@ public String getSegmentName() { */ @Override public double getStartTime() { + return getStartTime(new LogFileReadBudget(readLimits.getMaxExpandedBytes())); + } + + double getStartTime(LogFileReadBudget readBudget) { try { - ageOfJVMAtLogStart(); + ageOfJVMAtLogStart(readBudget); return startTime.getTimeStamp(); } catch (NullPointerException ex) { return Double.MAX_VALUE; @@ -89,10 +109,14 @@ public double getStartTime() { */ @Override public double getEndTime() { + return getEndTime(new LogFileReadBudget(readLimits.getMaxExpandedBytes())); + } + + double getEndTime(LogFileReadBudget readBudget) { try { - ageOfJVMAtLogEnd(); + ageOfJVMAtLogEnd(readBudget); return endTime.getTimeStamp(); - } catch (NullPointerException|IOException ex) { + } catch (NullPointerException ex) { return Double.MIN_VALUE; } } @@ -111,12 +135,18 @@ public int getSegmentIndex() { * @return A stream of lines from the file. */ public Stream stream() { + return stream(new LogFileReadBudget(readLimits.getMaxExpandedBytes())); + } + + Stream stream(LogFileReadBudget readBudget) { try { - return Files.lines(path); + return LogFileStreams.plainText( + path, + readLimits, + readBudget); } catch (IOException e) { - e.printStackTrace(); + throw new UncheckedIOException(e); } - return null; } /** @@ -127,20 +157,22 @@ public boolean isCurrent() { return current; } - private DateTimeStamp ageOfJVMAtLogStart() { + private DateTimeStamp ageOfJVMAtLogStart(LogFileReadBudget readBudget) { if (startTime == null) { - startTime = stream() - .map(DateTimeStamp::fromGCLogLine) - .filter(dateTimeStamp -> dateTimeStamp.hasTimeStamp() || dateTimeStamp.hasDateStamp()) - .findFirst() - .orElse(new DateTimeStamp(-1.0d)); + try (Stream lines = stream(readBudget)) { + startTime = lines + .map(DateTimeStamp::fromGCLogLine) + .filter(dateTimeStamp -> dateTimeStamp.hasTimeStamp() || dateTimeStamp.hasDateStamp()) + .findFirst() + .orElse(new DateTimeStamp(-1.0d)); + } } return startTime; } - private DateTimeStamp ageOfJVMAtLogEnd() throws IOException { + private DateTimeStamp ageOfJVMAtLogEnd(LogFileReadBudget readBudget) { if (endTime == null) { - endTime = tail(100).stream() + endTime = tail(100, readBudget).stream() .map(DateTimeStamp::fromGCLogLine) .filter(dateTimeStamp -> dateTimeStamp.hasTimeStamp() || dateTimeStamp.hasDateStamp()) .max(Comparator.comparing(dateTimeStamp -> dateTimeStamp != null ? dateTimeStamp.getTimeStamp() : 0)) @@ -158,54 +190,23 @@ public String toString() { return getSegmentName(); } - - // todo: implementation may be a bit ugly... - // https://codereview.stackexchange.com/questions/79039/get-the-tail-of-a-file-the-last-10-lines - // Tail is not a class, it's a method so the solution in stackoverflow isn't correct but the core - // could be used here as it's cleaner - private ArrayList tail(int numberOfLines) throws IOException { - - char LF = '\n'; - char CR = '\r'; - boolean foundEOL = false; - char eol = 0; - RandomAccessFile randomAccessFile = new RandomAccessFile(path.toFile(), "r"); - long currentPosition = randomAccessFile.length() - 1; - int linesFound = 0; - - while (currentPosition > 0 && !foundEOL) { - randomAccessFile.seek(currentPosition); - char character = (char) randomAccessFile.readByte(); - if (character == LF) { - eol = LF; - randomAccessFile.seek(currentPosition - 1); - character = (char) randomAccessFile.readByte(); - if (character == CR) - eol = CR; - foundEOL = true; - } else if (character == CR && !foundEOL) { - eol = CR; - foundEOL = true; - } else - currentPosition--; - } - - currentPosition = randomAccessFile.length() - 1; - - while (currentPosition > 0 && linesFound < numberOfLines) { - randomAccessFile.seek(--currentPosition); - char character = (char) randomAccessFile.readByte(); - if (eol == character) - linesFound++; + private List tail(int numberOfLines, LogFileReadBudget readBudget) { + try (Stream lines = stream(readBudget)) { + return lines.collect(tailCollector(numberOfLines)); } + } - ArrayList lines = new ArrayList<>(); - if (linesFound > 0) { - String line; - while ((line = randomAccessFile.readLine()) != null) { - lines.add(line); + private static Collector> tailCollector(int count) { + return Collector., List>of(ArrayDeque::new, (buffer, line) -> { + if (buffer.size() == count) { + buffer.pollFirst(); } - } - return lines; + buffer.add(line); + }, (buffer, list) -> { + while (list.size() < count && !buffer.isEmpty()) { + list.addFirst(buffer.pollLast()); + } + return list; + }, ArrayList::new); } } diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/GCLogFileZipSegment.java b/api/src/main/java/com/microsoft/gctoolkit/io/GCLogFileZipSegment.java index 8ffd0c09b..ee0fc1e21 100644 --- a/api/src/main/java/com/microsoft/gctoolkit/io/GCLogFileZipSegment.java +++ b/api/src/main/java/com/microsoft/gctoolkit/io/GCLogFileZipSegment.java @@ -4,19 +4,17 @@ import com.microsoft.gctoolkit.time.DateTimeStamp; -import java.io.BufferedReader; import java.io.IOException; -import java.io.InputStreamReader; +import java.io.UncheckedIOException; import java.nio.file.Path; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Comparator; import java.util.Deque; import java.util.List; +import java.util.Objects; import java.util.stream.Collector; import java.util.stream.Stream; -import java.util.zip.ZipEntry; -import java.util.zip.ZipFile; /** * A {@link RotatingGCLogFile} is made up of {@code GarbageCollectionLogFileSegment}s. Creating @@ -29,6 +27,9 @@ public class GCLogFileZipSegment implements LogFileSegment { private final Path path; private final String segmentName; + private final LogFileReadLimits readLimits; + private final ZipArchivePreflight.Index archiveIndex; + private final ZipArchivePreflight.Entry archiveEntry; private DateTimeStamp endTime = null; private DateTimeStamp startTime = null; @@ -38,8 +39,31 @@ public class GCLogFileZipSegment implements LogFileSegment { * @param segmentName name of first segment in zip file */ public GCLogFileZipSegment(Path path, String segmentName) { + this(path, segmentName, LogFileReadLimits.defaults()); + } + + /** + * Creates an archived log segment with explicit resource limits. + * + * @param path path to the ZIP file + * @param segmentName name of the entry in the ZIP file + * @param readLimits resource limits applied while streaming the segment + */ + public GCLogFileZipSegment(Path path, String segmentName, LogFileReadLimits readLimits) { + this(path, segmentName, readLimits, null, null); + } + + GCLogFileZipSegment( + Path path, + String segmentName, + LogFileReadLimits readLimits, + ZipArchivePreflight.Index archiveIndex, + ZipArchivePreflight.Entry archiveEntry) { this.path = path; this.segmentName = segmentName; + this.readLimits = Objects.requireNonNull(readLimits, "readLimits"); + this.archiveIndex = archiveIndex; + this.archiveEntry = archiveEntry; } /** @@ -54,21 +78,25 @@ public String getSegmentName() { return this.segmentName; } - private void ageOfJVMAtLogStart() { + private void ageOfJVMAtLogStart(LogFileReadBudget readBudget) { if (startTime == null) { - startTime = stream() - .filter(s -> ! s.contains(" file created ")) - .map(DateTimeStamp::fromGCLogLine) - .filter(dateTimeStamp -> dateTimeStamp.hasTimeStamp() || dateTimeStamp.hasDateStamp()) - .findFirst() - .orElse(new DateTimeStamp(-1.0d)); + try (Stream lines = stream(readBudget)) { + startTime = lines + .filter(s -> ! s.contains(" file created ")) + .map(DateTimeStamp::fromGCLogLine) + .filter(dateTimeStamp -> dateTimeStamp.hasTimeStamp() || dateTimeStamp.hasDateStamp()) + .findFirst() + .orElse(new DateTimeStamp(-1.0d)); + } } } - private DateTimeStamp ageOfJVMAtLogEnd() { + private DateTimeStamp ageOfJVMAtLogEnd(LogFileReadBudget readBudget) { if (endTime == null) { - List tail = stream(). - collect(tail(100)); + List tail; + try (Stream lines = stream(readBudget)) { + tail = lines.collect(tail(100)); + } endTime = tail.stream() .filter(line -> ! line.contains("Saved as")) .map(DateTimeStamp::fromGCLogLine) @@ -94,8 +122,12 @@ private DateTimeStamp ageOfJVMAtLogEnd() { @Override public double getStartTime() { + return getStartTime(new LogFileReadBudget(readLimits.getMaxExpandedBytes())); + } + + double getStartTime(LogFileReadBudget readBudget) { try { - ageOfJVMAtLogStart(); + ageOfJVMAtLogStart(readBudget); if ( startTime.hasTimeStamp()) return startTime.getTimeStamp(); else if ( startTime.hasDateStamp()) @@ -109,8 +141,12 @@ else if ( startTime.hasDateStamp()) @Override public double getEndTime() { + return getEndTime(new LogFileReadBudget(readLimits.getMaxExpandedBytes())); + } + + double getEndTime(LogFileReadBudget readBudget) { try { - ageOfJVMAtLogEnd(); + ageOfJVMAtLogEnd(readBudget); if ( endTime.hasTimeStamp()) return endTime.getTimeStamp(); else if ( endTime.hasDateStamp()) @@ -127,14 +163,27 @@ else if ( endTime.hasDateStamp()) * @return A stream of lines from the file. */ public Stream stream() { + return stream(new LogFileReadBudget(readLimits.getMaxExpandedBytes())); + } + + Stream stream(LogFileReadBudget readBudget) { try { - ZipFile file = new ZipFile(path.toFile()); - ZipEntry entry = file.getEntry(this.segmentName); - return new BufferedReader(new InputStreamReader(file.getInputStream(entry))).lines(); + if (archiveIndex != null && archiveEntry != null) { + return LogFileStreams.zipEntry( + path, + archiveIndex, + archiveEntry, + readLimits, + readBudget); + } + return LogFileStreams.zipEntry( + path, + segmentName, + readLimits, + readBudget); } catch (IOException e) { - e.printStackTrace(); + throw new UncheckedIOException(e); } - return new ArrayList().stream(); } /** diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/LimitedGZIPInputStream.java b/api/src/main/java/com/microsoft/gctoolkit/io/LimitedGZIPInputStream.java new file mode 100644 index 000000000..08cffde60 --- /dev/null +++ b/api/src/main/java/com/microsoft/gctoolkit/io/LimitedGZIPInputStream.java @@ -0,0 +1,279 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +package com.microsoft.gctoolkit.io; + +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.io.PushbackInputStream; +import java.nio.file.Path; +import java.util.zip.CRC32; +import java.util.zip.DataFormatException; +import java.util.zip.Inflater; +import java.util.zip.ZipException; + +final class LimitedGZIPInputStream extends InputStream { + + private static final int BUFFER_SIZE = 8192; + private static final int FLAG_HEADER_CRC = 2; + private static final int FLAG_EXTRA = 4; + private static final int FLAG_NAME = 8; + private static final int FLAG_COMMENT = 16; + private static final int FLAG_RESERVED = 224; + + private final CountingInputStream compressedSource; + private final PushbackInputStream source; + private final Inflater inflater = new Inflater(true); + private final CRC32 crc = new CRC32(); + private final CRC32 headerCrc = new CRC32(); + private final byte[] compressedBuffer = new byte[BUFFER_SIZE]; + private final byte[] singleByte = new byte[1]; + private final LogFileReadBudget budget; + private final LogFileReadLimits limits; + private final Path path; + + private boolean memberOpen; + private boolean endOfStream; + private boolean closed; + private int lastInputLength; + private long headerBytes; + private long memberExpandedBytes; + private int memberCount; + + LimitedGZIPInputStream( + InputStream inputStream, + LogFileReadBudget budget, + LogFileReadLimits limits, + Path path) { + compressedSource = new CountingInputStream( + inputStream, + limits.getMaxCompressedBytes(), + path); + source = new PushbackInputStream(compressedSource, BUFFER_SIZE); + this.budget = budget; + this.limits = limits; + this.path = path; + } + + @Override + public int read() throws IOException { + int count = read(singleByte, 0, 1); + return count == -1 ? -1 : Byte.toUnsignedInt(singleByte[0]); + } + + @Override + public int read(byte[] bytes, int offset, int length) throws IOException { + ensureOpen(); + if (length == 0) { + return 0; + } + + while (!endOfStream) { + if (!memberOpen && !openMember()) { + endOfStream = true; + return -1; + } + + try { + int permittedLength = budget.nextReadLength(length); + int count = inflater.inflate(bytes, offset, permittedLength); + if (count > 0) { + crc.update(bytes, offset, count); + budget.record(count, path, null); + memberExpandedBytes += count; + budget.recordCompressedExpansion( + count, + compressedSource.getBytesRead(), + limits, + path, + null); + checkCompressionRatio(); + return count; + } + if (inflater.finished()) { + finishMember(); + } else if (inflater.needsDictionary()) { + throw new ZipException("GZIP member requires a preset dictionary: " + path); + } else if (inflater.needsInput()) { + fillInflater(); + } else { + throw new ZipException("Unable to make progress while inflating " + path); + } + } catch (DataFormatException exception) { + ZipException failure = new ZipException("Invalid GZIP data: " + path); + failure.initCause(exception); + throw failure; + } + } + return -1; + } + + private boolean openMember() throws IOException { + int magic1 = source.read(); + if (magic1 == -1) { + return false; + } + if (++memberCount > limits.getMaxArchiveEntries()) { + throw new LogFileReadLimitExceededException( + LogFileReadLimitExceededException.LimitType.COMPRESSED_MEMBERS, + path, + null, + Integer.toString(limits.getMaxArchiveEntries()), + Integer.toString(memberCount)); + } + headerCrc.reset(); + headerCrc.update(magic1); + headerBytes = 1L; + checkHeaderBytes(); + int magic2 = readHeaderByte(); + if (magic1 != 0x1f || magic2 != 0x8b) { + throw new ZipException("Invalid GZIP header: " + path); + } + if (readHeaderByte() != 8) { + throw new ZipException("Unsupported GZIP compression method: " + path); + } + int flags = readHeaderByte(); + if ((flags & FLAG_RESERVED) != 0) { + throw new ZipException("Invalid GZIP flags: " + path); + } + skipHeaderBytes(6); + if ((flags & FLAG_EXTRA) != 0) { + int extraLength = readHeaderByte() | (readHeaderByte() << 8); + skipHeaderBytes(extraLength); + } + if ((flags & FLAG_NAME) != 0) { + skipZeroTerminatedHeaderField(); + } + if ((flags & FLAG_COMMENT) != 0) { + skipZeroTerminatedHeaderField(); + } + if ((flags & FLAG_HEADER_CRC) != 0) { + int expectedHeaderCrc = readStoredHeaderByte() | (readStoredHeaderByte() << 8); + if (expectedHeaderCrc != ((int) headerCrc.getValue() & 0xffff)) { + throw new ZipException("Corrupt GZIP header: " + path); + } + } + + inflater.reset(); + crc.reset(); + memberExpandedBytes = 0L; + lastInputLength = 0; + memberOpen = true; + return true; + } + + private void fillInflater() throws IOException { + int count = source.read(compressedBuffer); + if (count == -1) { + throw new EOFException("Unexpected end of GZIP member: " + path); + } + lastInputLength = count; + inflater.setInput(compressedBuffer, 0, count); + } + + private void finishMember() throws IOException { + int remaining = inflater.getRemaining(); + if (remaining > 0) { + source.unread(compressedBuffer, lastInputLength - remaining, remaining); + } + + long expectedCrc = readLittleEndianUnsignedInt(); + long expectedSize = readLittleEndianUnsignedInt(); + if (expectedCrc != crc.getValue()) { + throw new ZipException("Corrupt GZIP CRC: " + path); + } + if (expectedSize != (memberExpandedBytes & 0xffffffffL)) { + throw new ZipException("Corrupt GZIP size: " + path); + } + memberOpen = false; + } + + private void checkCompressionRatio() { + if (memberExpandedBytes <= limits.getCompressionRatioGraceBytes()) { + return; + } + long compressedBytes = headerBytes + inflater.getBytesRead(); + double ratio = compressedBytes == 0L + ? Double.POSITIVE_INFINITY + : (double) memberExpandedBytes / (double) compressedBytes; + if (ratio > limits.getMaxCompressionRatio()) { + throw new LogFileReadLimitExceededException( + LogFileReadLimitExceededException.LimitType.COMPRESSION_RATIO, + path, + null, + Double.toString(limits.getMaxCompressionRatio()), + Double.toString(ratio)); + } + } + + private int readHeaderByte() throws IOException { + int value = source.read(); + if (value == -1) { + throw new EOFException("Unexpected end of GZIP header: " + path); + } + headerBytes++; + checkHeaderBytes(); + headerCrc.update(value); + return value; + } + + private int readStoredHeaderByte() throws IOException { + int value = source.read(); + if (value == -1) { + throw new EOFException("Unexpected end of GZIP header: " + path); + } + headerBytes++; + checkHeaderBytes(); + return value; + } + + private void checkHeaderBytes() { + if (headerBytes > limits.getMaxGzipHeaderBytes()) { + throw new LogFileReadLimitExceededException( + LogFileReadLimitExceededException.LimitType.HEADER_BYTES, + path, + null, + Integer.toString(limits.getMaxGzipHeaderBytes()), + Long.toString(headerBytes)); + } + } + + private void skipHeaderBytes(int count) throws IOException { + for (int index = 0; index < count; index++) { + readHeaderByte(); + } + } + + private void skipZeroTerminatedHeaderField() throws IOException { + while (readHeaderByte() != 0) { + // Continue to the field terminator. + } + } + + private long readLittleEndianUnsignedInt() throws IOException { + long value = 0L; + for (int index = 0; index < 4; index++) { + int next = source.read(); + if (next == -1) { + throw new EOFException("Unexpected end of GZIP trailer: " + path); + } + value |= (long) next << (8 * index); + } + return value; + } + + private void ensureOpen() throws IOException { + if (closed) { + throw new IOException("Stream closed"); + } + } + + @Override + public void close() throws IOException { + if (!closed) { + closed = true; + inflater.end(); + source.close(); + } + } +} diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/LimitedInputStream.java b/api/src/main/java/com/microsoft/gctoolkit/io/LimitedInputStream.java new file mode 100644 index 000000000..0e6707bbf --- /dev/null +++ b/api/src/main/java/com/microsoft/gctoolkit/io/LimitedInputStream.java @@ -0,0 +1,95 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +package com.microsoft.gctoolkit.io; + +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; +import java.util.function.LongSupplier; + +final class LimitedInputStream extends FilterInputStream { + + private final LogFileReadBudget budget; + private final LogFileReadLimits limits; + private final Path path; + private final String archiveEntry; + private final LongSupplier compressedBytes; + private long expandedBytes; + private long recordedCompressedBytes; + + LimitedInputStream( + InputStream inputStream, + LogFileReadBudget budget, + LogFileReadLimits limits, + Path path, + String archiveEntry, + LongSupplier compressedBytes) { + super(inputStream); + this.budget = budget; + this.limits = limits; + this.path = path; + this.archiveEntry = archiveEntry; + this.compressedBytes = compressedBytes; + if (compressedBytes != null) { + budget.registerCompressedBytes(compressedBytes.getAsLong()); + } + } + + @Override + public int read() throws IOException { + int value = super.read(); + if (value != -1) { + record(1L); + } + return value; + } + + @Override + public int read(byte[] bytes, int offset, int length) throws IOException { + int permittedLength = budget.nextReadLength(length); + int count = super.read(bytes, offset, permittedLength); + if (count > 0) { + record(count); + } + return count; + } + + @Override + public long skip(long count) throws IOException { + long skipped = super.skip(budget.nextSkipLength(count)); + if (skipped > 0) { + record(skipped); + } + return skipped; + } + + private void record(long count) { + budget.record(count, path, archiveEntry); + expandedBytes += count; + if (compressedBytes == null) { + return; + } + long currentCompressedBytes = compressedBytes.getAsLong(); + if (currentCompressedBytes > recordedCompressedBytes) { + budget.registerCompressedBytes(currentCompressedBytes - recordedCompressedBytes); + recordedCompressedBytes = currentCompressedBytes; + } + budget.recordCompressedExpansion(count, -1L, limits, path, archiveEntry); + if (expandedBytes <= limits.getCompressionRatioGraceBytes()) { + return; + } + + double ratio = currentCompressedBytes == 0L + ? Double.POSITIVE_INFINITY + : (double) expandedBytes / (double) currentCompressedBytes; + if (ratio > limits.getMaxCompressionRatio()) { + throw new LogFileReadLimitExceededException( + LogFileReadLimitExceededException.LimitType.COMPRESSION_RATIO, + path, + archiveEntry, + Double.toString(limits.getMaxCompressionRatio()), + Double.toString(ratio)); + } + } +} diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/LogFileReadBudget.java b/api/src/main/java/com/microsoft/gctoolkit/io/LogFileReadBudget.java new file mode 100644 index 000000000..080e97242 --- /dev/null +++ b/api/src/main/java/com/microsoft/gctoolkit/io/LogFileReadBudget.java @@ -0,0 +1,96 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +package com.microsoft.gctoolkit.io; + +import java.nio.file.Path; +import java.util.concurrent.atomic.AtomicLong; + +final class LogFileReadBudget { + + private final long maximumBytes; + private final AtomicLong expandedBytes = new AtomicLong(); + private final AtomicLong ratioExpandedBytes = new AtomicLong(); + private final AtomicLong ratioCompressedBytes = new AtomicLong(); + + LogFileReadBudget(long maximumBytes) { + this.maximumBytes = maximumBytes; + } + + int nextReadLength(int requestedLength) { + long remaining = maximumBytes - expandedBytes.get(); + long detectableLength = remaining == Long.MAX_VALUE ? Long.MAX_VALUE : remaining + 1L; + return (int) Math.min(requestedLength, detectableLength); + } + + long nextSkipLength(long requestedLength) { + long remaining = maximumBytes - expandedBytes.get(); + long detectableLength = remaining == Long.MAX_VALUE ? Long.MAX_VALUE : remaining + 1L; + return Math.min(requestedLength, detectableLength); + } + + void record(long bytes, Path path, String archiveEntry) { + if (bytes <= 0) { + return; + } + while (true) { + long current = expandedBytes.get(); + if (bytes > maximumBytes - current) { + long observed = current == maximumBytes ? maximumBytes : current + bytes; + throw new LogFileReadLimitExceededException( + LogFileReadLimitExceededException.LimitType.EXPANDED_BYTES, + path, + archiveEntry, + Long.toString(maximumBytes), + Long.toString(observed)); + } + if (expandedBytes.compareAndSet(current, current + bytes)) { + return; + } + } + } + + void rejectDeclaredSize(long declaredBytes, Path path, String archiveEntry) { + long current = expandedBytes.get(); + if (declaredBytes > maximumBytes - current) { + throw new LogFileReadLimitExceededException( + LogFileReadLimitExceededException.LimitType.EXPANDED_BYTES, + path, + archiveEntry, + Long.toString(maximumBytes), + Long.toString(current + declaredBytes)); + } + } + + void registerCompressedBytes(long compressedBytes) { + if (compressedBytes > 0L) { + ratioCompressedBytes.addAndGet(compressedBytes); + } + } + + void recordCompressedExpansion( + long expanded, + long observedCompressedBytes, + LogFileReadLimits limits, + Path path, + String archiveEntry) { + if (observedCompressedBytes >= 0L) { + ratioCompressedBytes.accumulateAndGet(observedCompressedBytes, Math::max); + } + long totalExpanded = ratioExpandedBytes.addAndGet(expanded); + if (totalExpanded <= limits.getCompressionRatioGraceBytes()) { + return; + } + long totalCompressed = ratioCompressedBytes.get(); + double ratio = totalCompressed == 0L + ? Double.POSITIVE_INFINITY + : (double) totalExpanded / (double) totalCompressed; + if (ratio > limits.getMaxCompressionRatio()) { + throw new LogFileReadLimitExceededException( + LogFileReadLimitExceededException.LimitType.COMPRESSION_RATIO, + path, + archiveEntry, + Double.toString(limits.getMaxCompressionRatio()), + Double.toString(ratio)); + } + } +} diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/LogFileReadLimitExceededException.java b/api/src/main/java/com/microsoft/gctoolkit/io/LogFileReadLimitExceededException.java new file mode 100644 index 000000000..29395623e --- /dev/null +++ b/api/src/main/java/com/microsoft/gctoolkit/io/LogFileReadLimitExceededException.java @@ -0,0 +1,103 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +package com.microsoft.gctoolkit.io; + +import java.nio.file.Path; + +/** + * Indicates that a garbage collection log exceeded a configured resource limit while being read. + * + *

The exception is unchecked because stream I/O can fail lazily during a terminal stream + * operation, after {@link DataSource#stream()} has returned. + */ +public final class LogFileReadLimitExceededException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + /** The resource limit that was exceeded. */ + public enum LimitType { + EXPANDED_BYTES, + COMPRESSED_BYTES, + LINE_CHARACTERS, + COMPRESSION_RATIO, + ARCHIVE_ENTRIES, + COMPRESSED_MEMBERS, + HEADER_BYTES + } + + private final LimitType limitType; + private final Path path; + private final String archiveEntry; + private final String configuredLimit; + private final String observedValue; + + LogFileReadLimitExceededException( + LimitType limitType, + Path path, + String archiveEntry, + String configuredLimit, + String observedValue) { + super(message(limitType, path, archiveEntry, configuredLimit, observedValue)); + this.limitType = limitType; + this.path = path; + this.archiveEntry = archiveEntry; + this.configuredLimit = configuredLimit; + this.observedValue = observedValue; + } + + private static String message( + LimitType limitType, + Path path, + String archiveEntry, + String configuredLimit, + String observedValue) { + String source = archiveEntry == null ? path.toString() : path + "!" + archiveEntry; + return "Log file read limit exceeded for " + source + ": " + + limitType + " limit " + configuredLimit + ", observed " + observedValue; + } + + /** + * Returns the type of resource limit that was exceeded. + * + * @return limit type + */ + public LimitType getLimitType() { + return limitType; + } + + /** + * Returns the path being read when the limit was exceeded. + * + * @return source path + */ + public Path getPath() { + return path; + } + + /** + * Returns the ZIP entry being read, if any. + * + * @return ZIP entry name, or {@code null} for a top-level source + */ + public String getArchiveEntry() { + return archiveEntry; + } + + /** + * Returns the configured limit rendered in the units identified by {@link #getLimitType()}. + * + * @return configured limit + */ + public String getConfiguredLimit() { + return configuredLimit; + } + + /** + * Returns the observed value rendered in the units identified by {@link #getLimitType()}. + * + * @return observed value + */ + public String getObservedValue() { + return observedValue; + } +} diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/LogFileReadLimits.java b/api/src/main/java/com/microsoft/gctoolkit/io/LogFileReadLimits.java new file mode 100644 index 000000000..ffb8e83cc --- /dev/null +++ b/api/src/main/java/com/microsoft/gctoolkit/io/LogFileReadLimits.java @@ -0,0 +1,241 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +package com.microsoft.gctoolkit.io; + +import java.util.Objects; + +/** + * Resource limits applied while reading garbage collection logs. + * + *

Limits are enforced for every top-level stream operation. Applications that need to process + * larger trusted logs can supply higher finite values through the log-file constructors. + */ +public final class LogFileReadLimits { + + /** Default maximum number of bytes read from plaintext or emitted by decompression. */ + public static final long DEFAULT_MAX_EXPANDED_BYTES = 1024L * 1024L * 1024L; + + /** Default maximum number of bytes read from one compressed input. */ + public static final long DEFAULT_MAX_COMPRESSED_BYTES = 1024L * 1024L * 1024L; + + /** Default maximum number of decoded characters in one line. */ + public static final int DEFAULT_MAX_LINE_CHARACTERS = 1024 * 1024; + + /** Default maximum ratio of expanded bytes to compressed bytes. */ + public static final double DEFAULT_MAX_COMPRESSION_RATIO = 100.0d; + + /** Default expanded-byte threshold before compression-ratio enforcement begins. */ + public static final long DEFAULT_COMPRESSION_RATIO_GRACE_BYTES = 1024L * 1024L; + + /** Default maximum number of file entries in a rotating ZIP log. */ + public static final int DEFAULT_MAX_ARCHIVE_ENTRIES = 1024; + + /** Default maximum number of bytes in one GZIP member header. */ + public static final int DEFAULT_MAX_GZIP_HEADER_BYTES = 64 * 1024; + + private static final LogFileReadLimits DEFAULTS = new LogFileReadLimits( + DEFAULT_MAX_COMPRESSED_BYTES, + DEFAULT_MAX_EXPANDED_BYTES, + DEFAULT_MAX_LINE_CHARACTERS, + DEFAULT_MAX_COMPRESSION_RATIO, + DEFAULT_COMPRESSION_RATIO_GRACE_BYTES, + DEFAULT_MAX_ARCHIVE_ENTRIES, + DEFAULT_MAX_GZIP_HEADER_BYTES); + + private final long maxCompressedBytes; + private final long maxExpandedBytes; + private final int maxLineCharacters; + private final double maxCompressionRatio; + private final long compressionRatioGraceBytes; + private final int maxArchiveEntries; + private final int maxGzipHeaderBytes; + + /** + * Creates a set of finite log-file read limits. + * + * @param maxExpandedBytes maximum plaintext or expanded bytes per top-level stream operation + * @param maxLineCharacters maximum decoded characters in one line + * @param maxCompressionRatio maximum expanded-to-compressed byte ratio + * @param compressionRatioGraceBytes expanded bytes allowed before ratio enforcement begins + * @param maxArchiveEntries maximum file entries in a rotating ZIP log + */ + public LogFileReadLimits( + long maxExpandedBytes, + int maxLineCharacters, + double maxCompressionRatio, + long compressionRatioGraceBytes, + int maxArchiveEntries) { + this( + maxExpandedBytes, + maxExpandedBytes, + maxLineCharacters, + maxCompressionRatio, + compressionRatioGraceBytes, + maxArchiveEntries, + DEFAULT_MAX_GZIP_HEADER_BYTES); + } + + /** + * Creates a set of finite log-file read limits, including compressed-input limits. + * + * @param maxCompressedBytes maximum bytes read from one compressed input + * @param maxExpandedBytes maximum plaintext or expanded bytes per top-level stream operation + * @param maxLineCharacters maximum decoded characters in one line + * @param maxCompressionRatio maximum expanded-to-compressed byte ratio + * @param compressionRatioGraceBytes expanded bytes allowed before ratio enforcement begins + * @param maxArchiveEntries maximum ZIP entries, GZIP members, or rotating log segments + * @param maxGzipHeaderBytes maximum bytes in one GZIP member header + */ + public LogFileReadLimits( + long maxCompressedBytes, + long maxExpandedBytes, + int maxLineCharacters, + double maxCompressionRatio, + long compressionRatioGraceBytes, + int maxArchiveEntries, + int maxGzipHeaderBytes) { + if (maxCompressedBytes <= 0) { + throw new IllegalArgumentException("maxCompressedBytes must be positive"); + } + if (maxExpandedBytes <= 0) { + throw new IllegalArgumentException("maxExpandedBytes must be positive"); + } + if (maxLineCharacters <= 0) { + throw new IllegalArgumentException("maxLineCharacters must be positive"); + } + if (!Double.isFinite(maxCompressionRatio) || maxCompressionRatio <= 0.0d) { + throw new IllegalArgumentException("maxCompressionRatio must be positive and finite"); + } + if (compressionRatioGraceBytes <= 0) { + throw new IllegalArgumentException("compressionRatioGraceBytes must be positive"); + } + if (maxArchiveEntries <= 0) { + throw new IllegalArgumentException("maxArchiveEntries must be positive"); + } + if (maxGzipHeaderBytes <= 0) { + throw new IllegalArgumentException("maxGzipHeaderBytes must be positive"); + } + this.maxCompressedBytes = maxCompressedBytes; + this.maxExpandedBytes = maxExpandedBytes; + this.maxLineCharacters = maxLineCharacters; + this.maxCompressionRatio = maxCompressionRatio; + this.compressionRatioGraceBytes = compressionRatioGraceBytes; + this.maxArchiveEntries = maxArchiveEntries; + this.maxGzipHeaderBytes = maxGzipHeaderBytes; + } + + /** + * Returns the secure default limits. + * + * @return immutable default limits + */ + public static LogFileReadLimits defaults() { + return DEFAULTS; + } + + /** + * Returns the maximum number of bytes read from one compressed input. + * + * @return maximum compressed bytes + */ + public long getMaxCompressedBytes() { + return maxCompressedBytes; + } + + /** + * Returns the maximum number of plaintext or expanded bytes per top-level stream operation. + * + * @return maximum expanded bytes + */ + public long getMaxExpandedBytes() { + return maxExpandedBytes; + } + + /** + * Returns the maximum number of decoded characters in one line. + * + * @return maximum line characters + */ + public int getMaxLineCharacters() { + return maxLineCharacters; + } + + /** + * Returns the maximum expanded-to-compressed byte ratio. + * + * @return maximum compression ratio + */ + public double getMaxCompressionRatio() { + return maxCompressionRatio; + } + + /** + * Returns the expanded-byte threshold before compression-ratio enforcement begins. + * + * @return compression-ratio grace bytes + */ + public long getCompressionRatioGraceBytes() { + return compressionRatioGraceBytes; + } + + /** + * Returns the maximum number of entries accepted in a rotating ZIP log. + * + * @return maximum archive entries + */ + public int getMaxArchiveEntries() { + return maxArchiveEntries; + } + + /** + * Returns the maximum number of bytes accepted in one GZIP member header. + * + * @return maximum GZIP header bytes + */ + public int getMaxGzipHeaderBytes() { + return maxGzipHeaderBytes; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof LogFileReadLimits)) { + return false; + } + LogFileReadLimits that = (LogFileReadLimits) other; + return maxCompressedBytes == that.maxCompressedBytes + && maxExpandedBytes == that.maxExpandedBytes + && maxLineCharacters == that.maxLineCharacters + && Double.compare(maxCompressionRatio, that.maxCompressionRatio) == 0 + && compressionRatioGraceBytes == that.compressionRatioGraceBytes + && maxArchiveEntries == that.maxArchiveEntries + && maxGzipHeaderBytes == that.maxGzipHeaderBytes; + } + + @Override + public int hashCode() { + return Objects.hash( + maxCompressedBytes, + maxExpandedBytes, + maxLineCharacters, + maxCompressionRatio, + compressionRatioGraceBytes, + maxArchiveEntries, + maxGzipHeaderBytes); + } + + @Override + public String toString() { + return "LogFileReadLimits{" + + "maxCompressedBytes=" + maxCompressedBytes + + ", maxExpandedBytes=" + maxExpandedBytes + + ", maxLineCharacters=" + maxLineCharacters + + ", maxCompressionRatio=" + maxCompressionRatio + + ", compressionRatioGraceBytes=" + compressionRatioGraceBytes + + ", maxArchiveEntries=" + maxArchiveEntries + + ", maxGzipHeaderBytes=" + maxGzipHeaderBytes + + '}'; + } +} diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/LogFileStreams.java b/api/src/main/java/com/microsoft/gctoolkit/io/LogFileStreams.java new file mode 100644 index 000000000..81cefe514 --- /dev/null +++ b/api/src/main/java/com/microsoft/gctoolkit/io/LogFileStreams.java @@ -0,0 +1,180 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +package com.microsoft.gctoolkit.io; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.UncheckedIOException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.function.LongSupplier; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +final class LogFileStreams { + + private LogFileStreams() { + } + + static Stream plainText( + Path path, + LogFileReadLimits limits, + LogFileReadBudget budget) throws IOException { + return lines( + Files.newInputStream(path), + StandardCharsets.UTF_8, + limits, + budget, + path, + null, + null); + } + + static Stream gzip( + Path path, + LogFileReadLimits limits, + LogFileReadBudget budget) throws IOException { + LimitedGZIPInputStream gzip = new LimitedGZIPInputStream( + Files.newInputStream(path), + budget, + limits, + path); + try { + return decodedLines( + gzip, + Charset.defaultCharset(), + limits, + path, + null); + } catch (RuntimeException | Error failure) { + closeAfterFailure(gzip, failure); + throw failure; + } + } + + static Stream firstZipEntry( + Path path, + LogFileReadLimits limits, + LogFileReadBudget budget) throws IOException { + ZipArchivePreflight.Index index = + ZipArchivePreflight.validate(path, limits); + ZipArchivePreflight.Entry entry = index.firstFile(); + if (entry == null) { + throw new IOException("ZIP file contains no readable entries: " + path); + } + return zipEntry(path, index, entry, limits, budget); + } + + static Stream zipEntry( + Path path, + String entryName, + LogFileReadLimits limits, + LogFileReadBudget budget) throws IOException { + ZipArchivePreflight.Index index = + ZipArchivePreflight.validate(path, limits); + ZipArchivePreflight.Entry entry = index.find(entryName); + if (entry == null || entry.isDirectory()) { + throw new IOException("ZIP entry not found: " + path + "!" + entryName); + } + return zipEntry(path, index, entry, limits, budget); + } + + static Stream segment( + LogFileSegment segment, + LogFileReadLimits limits, + LogFileReadBudget budget) { + try { + if (segment instanceof GCLogFileZipSegment) { + return ((GCLogFileZipSegment) segment).stream(budget); + } + return plainText(segment.getPath(), limits, budget); + } catch (IOException exception) { + throw new UncheckedIOException(exception); + } + } + + private static Stream lines( + InputStream inputStream, + Charset charset, + LogFileReadLimits limits, + LogFileReadBudget budget, + Path path, + String archiveEntry, + LongSupplier compressedBytes) { + LimitedInputStream limited = new LimitedInputStream( + inputStream, + budget, + limits, + path, + archiveEntry, + compressedBytes); + return decodedLines(limited, charset, limits, path, archiveEntry); + } + + private static Stream decodedLines( + InputStream inputStream, + Charset charset, + LogFileReadLimits limits, + Path path, + String archiveEntry) { + BoundedLineSpliterator lines = new BoundedLineSpliterator( + new InputStreamReader(inputStream, charset), + limits.getMaxLineCharacters(), + path, + archiveEntry); + return StreamSupport.stream(lines, false).onClose(lines::closeUnchecked); + } + + static Stream zipEntry( + Path path, + ZipArchivePreflight.Index index, + ZipArchivePreflight.Entry entry, + LogFileReadLimits limits, + LogFileReadBudget budget) throws IOException { + budget.rejectDeclaredSize(entry.getExpandedSize(), path, entry.getName()); + if (entry.getExpandedSize() > limits.getCompressionRatioGraceBytes()) { + double ratio = entry.getCompressedSize() == 0L + ? Double.POSITIVE_INFINITY + : (double) entry.getExpandedSize() / (double) entry.getCompressedSize(); + if (ratio > limits.getMaxCompressionRatio()) { + throw new LogFileReadLimitExceededException( + LogFileReadLimitExceededException.LimitType.COMPRESSION_RATIO, + path, + entry.getName(), + Double.toString(limits.getMaxCompressionRatio()), + Double.toString(ratio)); + } + } + long archiveSliceLength = index.getCentralDirectoryOffset() - entry.getLocalHeaderOffset(); + ArchiveSliceInputStream archiveSlice = new ArchiveSliceInputStream( + path, + entry.getLocalHeaderOffset(), + archiveSliceLength); + try { + ValidatedZipEntryInputStream validated = + new ValidatedZipEntryInputStream(archiveSlice, entry, path); + return lines( + validated, + Charset.defaultCharset(), + limits, + budget, + path, + entry.getName(), + validated::getCompressedBytesRead); + } catch (IOException | RuntimeException | Error failure) { + closeAfterFailure(archiveSlice, failure); + throw failure; + } + } + + private static void closeAfterFailure(AutoCloseable closeable, Throwable failure) { + try { + closeable.close(); + } catch (Exception closeException) { + failure.addSuppressed(closeException); + } + } +} diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/RotatingGCLogFile.java b/api/src/main/java/com/microsoft/gctoolkit/io/RotatingGCLogFile.java index 6155cd1f8..8568b333f 100644 --- a/api/src/main/java/com/microsoft/gctoolkit/io/RotatingGCLogFile.java +++ b/api/src/main/java/com/microsoft/gctoolkit/io/RotatingGCLogFile.java @@ -2,23 +2,12 @@ // Licensed under the MIT License. package com.microsoft.gctoolkit.io; -import java.io.BufferedReader; import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.SequenceInputStream; -import java.io.UncheckedIOException; import java.nio.file.Path; -import java.util.Arrays; -import java.util.LinkedList; import java.util.List; import java.util.Objects; -import java.util.Vector; -import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; -import java.util.zip.ZipEntry; -import java.util.zip.ZipFile; /** * A collection of rotating GC log files. The collection will contain only those files that can be @@ -26,89 +15,49 @@ */ public class RotatingGCLogFile extends GCLogFile { - private static final Logger LOGGER = Logger.getLogger(RotatingGCLogFile.class.getName()); - /** * Use the given path to find rotating log files. If the path is a file, the file name is used to match * other files in the directory. If the path is a directory, all files in the directory are considered. * @param path the path to a rotating log file, or to a directory containing rotating log files. */ public RotatingGCLogFile(Path path) { - super(path); + this(path, LogFileReadLimits.defaults()); + } + + /** + * Use the given path to find rotating log files with explicit resource limits. + * + * @param path the path to a rotating log file, or to a directory containing rotating log files + * @param readLimits resource limits applied while inspecting and streaming the logs + */ + public RotatingGCLogFile(Path path, LogFileReadLimits readLimits) { + super(path, readLimits); } private RotatingLogFileMetadata metaData; public LogFileMetadata getMetaData() throws IOException { if ( metaData == null) - metaData = new RotatingLogFileMetadata(getPath()); + metaData = new RotatingLogFileMetadata(getPath(), getReadLimits()); return metaData; } @Override public Stream stream() throws IOException { - if ( getMetaData().isDirectory() || getMetaData().isPlainText() || getMetaData().isZip()) + LogFileReadBudget budget = new LogFileReadBudget(getReadLimits().getMaxExpandedBytes()); + LogFileMetadata metadata = getMetaData(); + if (metadata.isDirectory() || metadata.isPlainText() || metadata.isZip()) { + Stream segments = metaData.logFiles(budget); return Stream.concat( - getMetaData().logFiles() - .flatMap(LogFileSegment::stream) - .filter(Objects::nonNull) - .map(String::trim) - .filter(s -> s.length() > 0), + segments + .flatMap(segment -> LogFileStreams.segment(segment, getReadLimits(), budget)) + .filter(Objects::nonNull) + .map(String::trim) + .filter(s -> s.length() > 0), Stream.of(endOfData())); - else // yes, this is returning an empty stream. + } else { // yes, this is returning an empty stream. return Stream.of(endOfData()); - } - - private Stream stream(LogFileMetadata metadata, LinkedList segments) throws IOException { - //todo: find rolling files.... - if (metadata.isPlainText() || metadata.isDirectory()) { - switch (segments.size()) { - case 0: - String[] empty = new String[0]; - return Arrays.stream(empty); - case 1: - return segments.getFirst().stream(); - default: - // This code removes elements from the list of segments, so work on a copy. - LinkedList copySegments = new LinkedList<>(segments); - Stream allSegments = Stream.concat(copySegments.removeFirst().stream(), copySegments.removeFirst().stream()); - while (!copySegments.isEmpty()) - allSegments = Stream.concat(allSegments, copySegments.removeFirst().stream()); - return allSegments; - } - } else if (metadata.isZip()) { - return streamZipFile(); - } else if (metadata.isGZip()) { - throw new IOException("Unable to stream GZip files. Please unzip and retry"); - } - throw new IOException("Unrecognised file type"); - } - - @SuppressWarnings("resource") - private Stream streamZipFile() throws IOException { - ZipFile zipFile = new ZipFile(path.toFile()); - List entries = zipFile.stream().filter(entry -> !entry.isDirectory()).collect(Collectors.toList()); - Vector streams = new Vector<>(); - - try { - entries - .stream() - .map(entry -> { - try { - return zipFile.getInputStream(entry); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }) - .filter(Objects::nonNull) - .forEach(streams::add); - } catch (UncheckedIOException uioe) { - throw uioe.getCause(); } - - SequenceInputStream sequenceInputStream = new SequenceInputStream(streams.elements()); - - return new BufferedReader(new InputStreamReader(sequenceInputStream)).lines(); } /** diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/RotatingLogFileMetadata.java b/api/src/main/java/com/microsoft/gctoolkit/io/RotatingLogFileMetadata.java index 2b2764be5..c203b5a8b 100644 --- a/api/src/main/java/com/microsoft/gctoolkit/io/RotatingLogFileMetadata.java +++ b/api/src/main/java/com/microsoft/gctoolkit/io/RotatingLogFileMetadata.java @@ -3,17 +3,16 @@ package com.microsoft.gctoolkit.io; import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.Comparator; import java.util.LinkedList; import java.util.List; -import java.util.logging.Level; +import java.util.Objects; import java.util.logging.Logger; import java.util.stream.Stream; -import java.util.zip.ZipEntry; -import java.util.zip.ZipFile; import static java.util.stream.Collectors.toList; @@ -25,6 +24,7 @@ public class RotatingLogFileMetadata extends LogFileMetadata { private static final Logger LOG = Logger.getLogger(RotatingLogFileMetadata.class.getName()); private List segments; + private final LogFileReadLimits readLimits; /** * Creates metadata for a rotating garbage collection log source. @@ -33,7 +33,19 @@ public class RotatingLogFileMetadata extends LogFileMetadata { * @throws IOException if the path cannot be inspected */ public RotatingLogFileMetadata(Path path) throws IOException { + this(path, LogFileReadLimits.defaults()); + } + + /** + * Creates metadata for a rotating garbage collection log source with explicit resource limits. + * + * @param path path to a rotating log file, archive, or directory + * @param readLimits resource limits applied while inspecting log segments + * @throws IOException if the path cannot be inspected + */ + public RotatingLogFileMetadata(Path path, LogFileReadLimits readLimits) throws IOException { super(path); + this.readLimits = Objects.requireNonNull(readLimits, "readLimits"); } /** @@ -42,11 +54,15 @@ public RotatingLogFileMetadata(Path path) throws IOException { * @return a stream of ordered log segments */ public Stream logFiles() { + return logFiles(new LogFileReadBudget(readLimits.getMaxExpandedBytes())); + } + + Stream logFiles(LogFileReadBudget inspectionBudget) { if ( segments == null) { if ( isPlainText() || isDirectory()) - findSegments(); + findSegments(inspectionBudget); else if ( isZip()) - findZIPSegments(); + findZIPSegments(inspectionBudget); else { LOG.warning("unknown log file format"); segments = new ArrayList<>(); @@ -55,17 +71,23 @@ else if ( isZip()) return segments.stream(); } - private void findZIPSegments() { - try (var zipfile = new ZipFile(getPath().toFile())) { - segments = zipfile.stream() - .filter(zipEntry -> !zipEntry.isDirectory()) - .map(ZipEntry::getName) - .map(name -> new GCLogFileZipSegment(getPath(),name)) - .collect(toList()); + private void findZIPSegments(LogFileReadBudget inspectionBudget) { + ZipArchivePreflight.Index index; + try { + index = ZipArchivePreflight.validate(getPath(), readLimits); } catch (IOException ioe) { - LOG.warning(ioe.getMessage()); + throw new UncheckedIOException(ioe); } - orderSegments(); + segments = index.getEntries().stream() + .filter(entry -> !entry.isDirectory()) + .map(entry -> new GCLogFileZipSegment( + getPath(), + entry.getName(), + readLimits, + index, + entry)) + .collect(toList()); + orderSegments(inspectionBudget); } /** @@ -76,9 +98,9 @@ private void findZIPSegments() { public int getNumberOfFiles() { if ( this.segments == null) if ( isZip()) - findZIPSegments(); + findZIPSegments(new LogFileReadBudget(readLimits.getMaxExpandedBytes())); else - findSegments(); + findSegments(new LogFileReadBudget(readLimits.getMaxExpandedBytes())); return this.segments.size(); } @@ -129,52 +151,77 @@ else if ( bits[bits.length - 1].matches("\\d+$")) return base.toString(); } - private void findSegments() { - segments = new ArrayList<>(); - try { + private void findSegments(LogFileReadBudget inspectionBudget) { + try (Stream paths = Files.list(isDirectory() ? getPath() : getPath().getParent())) { + Stream matchingPaths = paths; if (isDirectory()) { - Files.list(getPath()).map(GCLogFileSegment::new).forEach(segments::add); + matchingPaths = paths; + } else { + matchingPaths = paths.filter( + file -> file.getFileName().toString().startsWith(getRootPattern())); } - else { - Files.list(getPath().getParent()) - .filter(file -> file.getFileName().toString().startsWith(getRootPattern())) - .map(p -> new GCLogFileSegment(p)).forEach(segments::add); + List segmentPaths = matchingPaths + .limit((long) readLimits.getMaxArchiveEntries() + 1L) + .collect(toList()); + if (segmentPaths.size() > readLimits.getMaxArchiveEntries()) { + throw new LogFileReadLimitExceededException( + LogFileReadLimitExceededException.LimitType.ARCHIVE_ENTRIES, + getPath(), + null, + Integer.toString(readLimits.getMaxArchiveEntries()), + Integer.toString(segmentPaths.size())); } + segments = segmentPaths.stream() + .map(path -> new GCLogFileSegment(path, readLimits)) + .collect(toList()); } catch (IOException ioe) { - LOG.log(Level.WARNING,"Unable to find log segments.", ioe); + throw new UncheckedIOException(ioe); } - orderSegments(); + orderSegments(inspectionBudget); } - private void orderSegments() { + private void orderSegments(LogFileReadBudget inspectionBudget) { if (segments.size() < 2) return; - LinkedList orderedList = new LinkedList<>(); - List workingList = new ArrayList<>(); - workingList.addAll(segments); - - // Find current String basePattern = getRootPattern(); - LogFileSegment current = workingList.stream() + LogFileSegment current = segments.stream() .filter( segment -> segment.getSegmentName().endsWith(basePattern) || segment.getSegmentName().endsWith(".current")) .findFirst().get(); + LinkedList orderedList = new LinkedList<>(); orderedList.addLast(current); - workingList = removeIneligibleSegments (workingList, current); - while ( ! workingList.isEmpty()) { - current = workingList.stream() - .max(Comparator.comparing(LogFileSegment::getEndTime)) - .get(); - orderedList.addFirst(current); - workingList = removeIneligibleSegments (workingList, current); + double nextStartTime = getStartTime(current, inspectionBudget); + List candidates = segments.stream() + .filter(segment -> segment != current) + .sorted(Comparator.comparing( + (LogFileSegment segment) -> getEndTime(segment, inspectionBudget)) + .reversed()) + .collect(toList()); + for (LogFileSegment candidate : candidates) { + if (getEndTime(candidate, inspectionBudget) <= nextStartTime) { + orderedList.addFirst(candidate); + nextStartTime = getStartTime(candidate, inspectionBudget); + } } segments = orderedList; } - private List removeIneligibleSegments(final List logFileSegments, final LogFileSegment current) { - return logFileSegments.stream() - .filter( segment -> segment.getEndTime() <= current.getStartTime()) - .collect(toList()); + private static double getStartTime( + LogFileSegment segment, + LogFileReadBudget inspectionBudget) { + if (segment instanceof GCLogFileZipSegment) { + return ((GCLogFileZipSegment) segment).getStartTime(inspectionBudget); + } + return ((GCLogFileSegment) segment).getStartTime(inspectionBudget); + } + + private static double getEndTime( + LogFileSegment segment, + LogFileReadBudget inspectionBudget) { + if (segment instanceof GCLogFileZipSegment) { + return ((GCLogFileZipSegment) segment).getEndTime(inspectionBudget); + } + return ((GCLogFileSegment) segment).getEndTime(inspectionBudget); } } diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/SingleGCLogFile.java b/api/src/main/java/com/microsoft/gctoolkit/io/SingleGCLogFile.java index b3020b33e..dddaee0cb 100644 --- a/api/src/main/java/com/microsoft/gctoolkit/io/SingleGCLogFile.java +++ b/api/src/main/java/com/microsoft/gctoolkit/io/SingleGCLogFile.java @@ -2,18 +2,10 @@ // Licensed under the MIT License. package com.microsoft.gctoolkit.io; -import java.io.BufferedInputStream; -import java.io.BufferedReader; import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.file.Files; import java.nio.file.Path; import java.util.Objects; -import java.util.logging.Logger; import java.util.stream.Stream; -import java.util.zip.GZIPInputStream; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; /** * A single GC log file. If the file is a zip or gzip file, @@ -21,23 +13,30 @@ */ public class SingleGCLogFile extends GCLogFile { - private static final Logger LOGGER = Logger.getLogger(SingleGCLogFile.class.getName()); + private SingleLogFileMetadata metadata = null; /** * Constructor for a single, GC log file. * @param path The path to the log file. */ - - private SingleLogFileMetadata metadata = null; - public SingleGCLogFile(Path path) { - super(path); + this(path, LogFileReadLimits.defaults()); + } + + /** + * Constructor for a single GC log file with explicit resource limits. + * + * @param path the path to the log file + * @param readLimits resource limits applied while streaming the log + */ + public SingleGCLogFile(Path path, LogFileReadLimits readLimits) { + super(path, readLimits); } @Override public LogFileMetadata getMetaData() throws IOException { if (metadata == null) { - metadata = new SingleLogFileMetadata(path); + metadata = new SingleLogFileMetadata(path, getReadLimits()); } return metadata; } @@ -48,16 +47,17 @@ public Stream stream() throws IOException { } private Stream stream(LogFileMetadata metadata) throws IOException { - Stream stream = null; + LogFileReadBudget budget = new LogFileReadBudget(getReadLimits().getMaxExpandedBytes()); + Stream stream; if (metadata.isPlainText()) { - stream = Files.lines(metadata.getPath()); + stream = LogFileStreams.plainText(metadata.getPath(), getReadLimits(), budget); } else if (metadata.isZip()) { - stream = streamZipFile(metadata.getPath()); + stream = LogFileStreams.firstZipEntry(metadata.getPath(), getReadLimits(), budget); } else if (metadata.isGZip()) { - stream = streamGZipFile(metadata.getPath()); + stream = LogFileStreams.gzip(metadata.getPath(), getReadLimits(), budget); + } else { + throw new IOException("Unable to read " + path); } - if ( stream == null) - throw new IOException("Unable to read " + path.toString()); return Stream.concat(stream .filter(Objects::nonNull) .filter(line -> ! line.isBlank()) @@ -66,19 +66,4 @@ private Stream stream(LogFileMetadata metadata) throws IOException { ,Stream.of(endOfData())); } - - private static Stream streamZipFile(Path path) throws IOException { - ZipInputStream zipStream = new ZipInputStream(Files.newInputStream(path)); - ZipEntry entry; - do { - entry = zipStream.getNextEntry(); - } while (entry != null && entry.isDirectory()); - return new BufferedReader(new InputStreamReader(new BufferedInputStream(zipStream))).lines(); - } - - private static Stream streamGZipFile(Path path) throws IOException { - GZIPInputStream gzipStream = new GZIPInputStream(Files.newInputStream(path)); - return new BufferedReader(new InputStreamReader(new BufferedInputStream(gzipStream))).lines(); - } - } diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/SingleLogFileMetadata.java b/api/src/main/java/com/microsoft/gctoolkit/io/SingleLogFileMetadata.java index 8be508c78..3737955f2 100644 --- a/api/src/main/java/com/microsoft/gctoolkit/io/SingleLogFileMetadata.java +++ b/api/src/main/java/com/microsoft/gctoolkit/io/SingleLogFileMetadata.java @@ -24,8 +24,12 @@ public class SingleLogFileMetadata extends LogFileMetadata { * @throws IOException if the path cannot be inspected */ public SingleLogFileMetadata(Path path) throws IOException { + this(path, LogFileReadLimits.defaults()); + } + + SingleLogFileMetadata(Path path, LogFileReadLimits readLimits) throws IOException { super(path); - this.logFile = new GCLogFileSegment(path); + this.logFile = new GCLogFileSegment(path, readLimits); } /** diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/ValidatedZipEntryInputStream.java b/api/src/main/java/com/microsoft/gctoolkit/io/ValidatedZipEntryInputStream.java new file mode 100644 index 000000000..5adbc93d0 --- /dev/null +++ b/api/src/main/java/com/microsoft/gctoolkit/io/ValidatedZipEntryInputStream.java @@ -0,0 +1,277 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +package com.microsoft.gctoolkit.io; + +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.zip.CRC32; +import java.util.zip.ZipEntry; +import java.util.zip.ZipException; +import java.util.zip.ZipInputStream; + +final class ValidatedZipEntryInputStream extends InputStream { + + private static final long LOCAL_FILE_HEADER = 0x04034b50L; + private static final long DATA_DESCRIPTOR = 0x08074b50L; + private static final int DATA_DESCRIPTOR_FLAG = 8; + + private final InputStream entryInputStream; + private final TrackingZipInputStream trackingZipInputStream; + private final ZipArchivePreflight.Entry expectedEntry; + private final Path path; + private final CRC32 crc = new CRC32(); + private final byte[] singleByte = new byte[1]; + private long expandedBytes; + private boolean validated; + + ValidatedZipEntryInputStream( + InputStream inputStream, + ZipArchivePreflight.Entry expectedEntry, + Path path) throws IOException { + this.expectedEntry = expectedEntry; + this.path = path; + TrackingZipInputStream tracking = null; + try { + if (expectedEntry.getMethod() == ZipEntry.STORED) { + entryInputStream = new StoredEntryInputStream(inputStream, expectedEntry, path); + } else { + tracking = new TrackingZipInputStream(inputStream); + ZipEntry actualEntry = tracking.getNextEntry(); + if (actualEntry == null + || actualEntry.isDirectory() + || actualEntry.getMethod() != expectedEntry.getMethod() + || !expectedEntry.getName().equals(actualEntry.getName())) { + throw new ZipException( + "ZIP local header does not match central directory: " + path); + } + entryInputStream = tracking; + } + trackingZipInputStream = tracking; + } catch (IOException | RuntimeException | Error failure) { + try { + inputStream.close(); + } catch (IOException closeException) { + failure.addSuppressed(closeException); + } + throw failure; + } + } + + long getCompressedBytesRead() { + if (expectedEntry.getMethod() == ZipEntry.STORED) { + return expandedBytes; + } + return trackingZipInputStream.getCompressedBytesRead(); + } + + @Override + public int read() throws IOException { + int count = read(singleByte, 0, 1); + return count == -1 ? -1 : Byte.toUnsignedInt(singleByte[0]); + } + + @Override + public int read(byte[] bytes, int offset, int length) throws IOException { + int count = entryInputStream.read(bytes, offset, length); + if (count > 0) { + crc.update(bytes, offset, count); + expandedBytes += count; + } else if (count == -1) { + validate(); + } + return count; + } + + private void validate() throws ZipException { + if (validated) { + return; + } + validated = true; + if (expandedBytes != expectedEntry.getExpandedSize()) { + throw new ZipException("ZIP expanded size does not match central directory: " + path); + } + if (getCompressedBytesRead() != expectedEntry.getCompressedSize()) { + throw new ZipException("ZIP compressed size does not match central directory: " + path); + } + if (crc.getValue() != expectedEntry.getCrc()) { + throw new ZipException("ZIP CRC does not match central directory: " + path); + } + } + + @Override + public void close() throws IOException { + entryInputStream.close(); + } + + private static final class TrackingZipInputStream extends ZipInputStream { + + private TrackingZipInputStream(InputStream inputStream) { + super(inputStream); + } + + private long getCompressedBytesRead() { + return inf.getBytesRead(); + } + } + + private static final class StoredEntryInputStream extends InputStream { + + private final InputStream inputStream; + private final ZipArchivePreflight.Entry expectedEntry; + private final Path path; + private final boolean hasDescriptor; + private long remaining; + private boolean descriptorValidated; + + private StoredEntryInputStream( + InputStream inputStream, + ZipArchivePreflight.Entry expectedEntry, + Path path) throws IOException { + this.inputStream = inputStream; + this.expectedEntry = expectedEntry; + this.path = path; + if (readUnsignedInt(inputStream, path) != LOCAL_FILE_HEADER) { + throw new ZipException("ZIP local header not found: " + path); + } + ValidatedZipEntryInputStream.skip(inputStream, 2, path); + int flags = readUnsignedShort(inputStream, path); + int method = readUnsignedShort(inputStream, path); + ValidatedZipEntryInputStream.skip(inputStream, 4, path); + long localCrc = readUnsignedInt(inputStream, path); + long localCompressedSize = readUnsignedInt(inputStream, path); + long localExpandedSize = readUnsignedInt(inputStream, path); + int nameLength = readUnsignedShort(inputStream, path); + int extraLength = readUnsignedShort(inputStream, path); + byte[] name = readBytes(inputStream, nameLength, path); + ValidatedZipEntryInputStream.skip(inputStream, extraLength, path); + + if (method != ZipEntry.STORED + || flags != expectedEntry.getFlags() + || !new String(name, StandardCharsets.UTF_8).equals(expectedEntry.getName())) { + throw new ZipException( + "ZIP local header does not match central directory: " + path); + } + hasDescriptor = (flags & DATA_DESCRIPTOR_FLAG) != 0; + if (!hasDescriptor) { + if (localCrc != expectedEntry.getCrc() + || (localCompressedSize != 0xffffffffL + && localCompressedSize != expectedEntry.getCompressedSize()) + || (localExpandedSize != 0xffffffffL + && localExpandedSize != expectedEntry.getExpandedSize())) { + throw new ZipException( + "ZIP local header values do not match central directory: " + path); + } + } + remaining = expectedEntry.getCompressedSize(); + } + + @Override + public int read() throws IOException { + byte[] oneByte = new byte[1]; + int count = read(oneByte, 0, 1); + return count == -1 ? -1 : Byte.toUnsignedInt(oneByte[0]); + } + + @Override + public int read(byte[] bytes, int offset, int length) throws IOException { + if (remaining == 0L) { + validateDescriptor(); + return -1; + } + int count = inputStream.read(bytes, offset, (int) Math.min(length, remaining)); + if (count == -1) { + throw new EOFException("Unexpected end of stored ZIP entry: " + path); + } + remaining -= count; + return count; + } + + private void validateDescriptor() throws IOException { + if (!hasDescriptor || descriptorValidated) { + return; + } + descriptorValidated = true; + long crcOrSignature = readUnsignedInt(inputStream, path); + long descriptorCrc = crcOrSignature == DATA_DESCRIPTOR + ? readUnsignedInt(inputStream, path) + : crcOrSignature; + long compressedSize = expectedEntry.isZip64Size() + ? readLong(inputStream, path) + : readUnsignedInt(inputStream, path); + long expandedSize = expectedEntry.isZip64Size() + ? readLong(inputStream, path) + : readUnsignedInt(inputStream, path); + if (descriptorCrc != expectedEntry.getCrc() + || compressedSize != expectedEntry.getCompressedSize() + || expandedSize != expectedEntry.getExpandedSize()) { + throw new ZipException( + "ZIP data descriptor does not match central directory: " + path); + } + } + + @Override + public void close() throws IOException { + inputStream.close(); + } + } + + private static int readUnsignedShort(InputStream inputStream, Path path) throws IOException { + return readByte(inputStream, path) | (readByte(inputStream, path) << 8); + } + + private static long readUnsignedInt(InputStream inputStream, Path path) throws IOException { + return (long) readByte(inputStream, path) + | ((long) readByte(inputStream, path) << 8) + | ((long) readByte(inputStream, path) << 16) + | ((long) readByte(inputStream, path) << 24); + } + + private static long readLong(InputStream inputStream, Path path) throws IOException { + long value = readUnsignedInt(inputStream, path) + | (readUnsignedInt(inputStream, path) << 32); + if (value < 0L) { + throw new ZipException("ZIP64 value is too large: " + path); + } + return value; + } + + private static int readByte(InputStream inputStream, Path path) throws IOException { + int value = inputStream.read(); + if (value == -1) { + throw new EOFException("Unexpected end of ZIP metadata: " + path); + } + return value; + } + + private static byte[] readBytes( + InputStream inputStream, + int length, + Path path) throws IOException { + byte[] bytes = new byte[length]; + int offset = 0; + while (offset < length) { + int count = inputStream.read(bytes, offset, length - offset); + if (count == -1) { + throw new EOFException("Unexpected end of ZIP metadata: " + path); + } + offset += count; + } + return bytes; + } + + private static void skip(InputStream inputStream, int count, Path path) throws IOException { + int remaining = count; + while (remaining > 0) { + long skipped = inputStream.skip(remaining); + if (skipped == 0L) { + readByte(inputStream, path); + remaining--; + } else { + remaining -= (int) skipped; + } + } + } +} diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/ZipArchivePreflight.java b/api/src/main/java/com/microsoft/gctoolkit/io/ZipArchivePreflight.java new file mode 100644 index 000000000..613fd1616 --- /dev/null +++ b/api/src/main/java/com/microsoft/gctoolkit/io/ZipArchivePreflight.java @@ -0,0 +1,490 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +package com.microsoft.gctoolkit.io; + +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.zip.ZipEntry; +import java.util.zip.ZipException; + +final class ZipArchivePreflight { + + private static final long END_OF_CENTRAL_DIRECTORY = 0x06054b50L; + private static final long ZIP64_END_OF_CENTRAL_DIRECTORY = 0x06064b50L; + private static final long ZIP64_END_OF_CENTRAL_DIRECTORY_LOCATOR = 0x07064b50L; + private static final long CENTRAL_DIRECTORY_ENTRY = 0x02014b50L; + private static final long CENTRAL_DIRECTORY_ARCHIVE_EXTRA_DATA = 0x08064b50L; + private static final long CENTRAL_DIRECTORY_DIGITAL_SIGNATURE = 0x05054b50L; + private static final int ZIP64_EXTRA_FIELD = 1; + private static final int MAXIMUM_COMMENT_LENGTH = 0xffff; + private static final int END_OF_CENTRAL_DIRECTORY_SIZE = 22; + + private ZipArchivePreflight() { + } + + static Index validate(Path path, LogFileReadLimits limits) throws IOException { + try (RandomAccessFile file = new RandomAccessFile(path.toFile(), "r")) { + if (file.length() > limits.getMaxCompressedBytes()) { + throw new LogFileReadLimitExceededException( + LogFileReadLimitExceededException.LimitType.COMPRESSED_BYTES, + path, + null, + Long.toString(limits.getMaxCompressedBytes()), + Long.toString(file.length())); + } + Directory directory = readDirectory(file, path); + if (directory.entryCount > limits.getMaxArchiveEntries()) { + throw entryLimit(path, limits.getMaxArchiveEntries(), directory.entryCount); + } + + long directoryEnd = directory.offset + directory.size; + file.seek(directory.offset); + List entries = new ArrayList<>((int) directory.entryCount); + long recordCount = 0L; + long maximumRecords = (long) limits.getMaxArchiveEntries() + 2L; + while (file.getFilePointer() < directoryEnd) { + if (++recordCount > maximumRecords) { + throw entryLimit(path, (int) maximumRecords, recordCount); + } + long signature = readUnsignedInt(file); + if (signature == CENTRAL_DIRECTORY_DIGITAL_SIGNATURE) { + int signatureLength = readUnsignedShort(file); + skip(file, signatureLength, directoryEnd, path); + break; + } + if (signature == CENTRAL_DIRECTORY_ARCHIVE_EXTRA_DATA) { + long extraDataLength = readUnsignedInt(file); + skip(file, extraDataLength, directoryEnd, path); + continue; + } + if (signature != CENTRAL_DIRECTORY_ENTRY) { + throw new ZipException("Invalid ZIP central directory entry: " + path); + } + if (entries.size() == limits.getMaxArchiveEntries()) { + throw entryLimit(path, limits.getMaxArchiveEntries(), entries.size() + 1L); + } + entries.add(readEntry(file, path, directory.offset, directoryEnd)); + } + if (file.getFilePointer() != directoryEnd) { + throw new ZipException("Invalid ZIP central directory size: " + path); + } + if (entries.size() != directory.entryCount) { + throw new ZipException("ZIP central directory entry count mismatch: " + path); + } + return new Index(directory.offset, entries); + } + } + + private static Directory readDirectory(RandomAccessFile file, Path path) throws IOException { + long endOffset = findEndOfCentralDirectory(file); + file.seek(endOffset + 4); + int diskNumber = readUnsignedShort(file); + int centralDirectoryDisk = readUnsignedShort(file); + long entriesOnDisk = readUnsignedShort(file); + long totalEntries = readUnsignedShort(file); + long centralDirectorySize = readUnsignedInt(file); + long centralDirectoryOffset = readUnsignedInt(file); + + if (diskNumber != 0 || centralDirectoryDisk != 0 || entriesOnDisk != totalEntries) { + throw new ZipException("Split ZIP archives are not supported: " + path); + } + if (totalEntries == 0xffffL + || centralDirectorySize == 0xffffffffL + || centralDirectoryOffset == 0xffffffffL) { + return readZip64Directory(file, path, endOffset); + } + return checkedDirectory( + path, + file.length(), + totalEntries, + centralDirectorySize, + centralDirectoryOffset); + } + + private static Entry readEntry( + RandomAccessFile file, + Path path, + long centralDirectoryOffset, + long directoryEnd) throws IOException { + skip(file, 4, directoryEnd, path); + int flags = readUnsignedShort(file); + int method = readUnsignedShort(file); + skip(file, 4, directoryEnd, path); + long crc = readUnsignedInt(file); + long compressedSize = readUnsignedInt(file); + long expandedSize = readUnsignedInt(file); + boolean zip64Size = compressedSize == 0xffffffffL || expandedSize == 0xffffffffL; + int nameLength = readUnsignedShort(file); + int extraLength = readUnsignedShort(file); + int commentLength = readUnsignedShort(file); + int diskStart = readUnsignedShort(file); + skip(file, 6, directoryEnd, path); + long localHeaderOffset = readUnsignedInt(file); + + byte[] nameBytes = new byte[nameLength]; + file.readFully(nameBytes); + byte[] extra = new byte[extraLength]; + file.readFully(extra); + skip(file, commentLength, directoryEnd, path); + + long[] zip64 = parseZip64Extra( + extra, + expandedSize == 0xffffffffL, + compressedSize == 0xffffffffL, + localHeaderOffset == 0xffffffffL, + diskStart == 0xffff, + path); + int zip64Index = 0; + if (expandedSize == 0xffffffffL) { + expandedSize = zip64[zip64Index++]; + } + if (compressedSize == 0xffffffffL) { + compressedSize = zip64[zip64Index++]; + } + if (localHeaderOffset == 0xffffffffL) { + localHeaderOffset = zip64[zip64Index++]; + } + if (diskStart == 0xffff) { + diskStart = Math.toIntExact(zip64[zip64Index]); + } + if (diskStart != 0) { + throw new ZipException("Split ZIP entries are not supported: " + path); + } + if (localHeaderOffset < 0L || localHeaderOffset >= centralDirectoryOffset) { + throw new ZipException("Invalid ZIP local header offset: " + path); + } + if ((flags & 1) != 0) { + throw new ZipException("Encrypted ZIP entries are not supported: " + path); + } + if (method != ZipEntry.STORED && method != ZipEntry.DEFLATED) { + throw new ZipException("Unsupported ZIP compression method: " + method); + } + + String name = new String(nameBytes, StandardCharsets.UTF_8); + return new Entry( + name, + name.endsWith("/"), + flags, + method, + crc, + compressedSize, + expandedSize, + localHeaderOffset, + zip64Size); + } + + private static long[] parseZip64Extra( + byte[] extra, + boolean expandedSizeRequired, + boolean compressedSizeRequired, + boolean offsetRequired, + boolean diskRequired, + Path path) throws ZipException { + int requiredValues = (expandedSizeRequired ? 1 : 0) + + (compressedSizeRequired ? 1 : 0) + + (offsetRequired ? 1 : 0) + + (diskRequired ? 1 : 0); + if (requiredValues == 0) { + return new long[0]; + } + + int offset = 0; + while (offset <= extra.length - 4) { + int id = unsignedShort(extra, offset); + int size = unsignedShort(extra, offset + 2); + offset += 4; + if (size > extra.length - offset) { + throw new ZipException("Invalid ZIP extra field: " + path); + } + if (id == ZIP64_EXTRA_FIELD) { + List values = new ArrayList<>(requiredValues); + int valueOffset = offset; + int fieldEnd = offset + size; + if (expandedSizeRequired) { + values.add(littleEndianLong(extra, valueOffset, fieldEnd, path)); + valueOffset += 8; + } + if (compressedSizeRequired) { + values.add(littleEndianLong(extra, valueOffset, fieldEnd, path)); + valueOffset += 8; + } + if (offsetRequired) { + values.add(littleEndianLong(extra, valueOffset, fieldEnd, path)); + valueOffset += 8; + } + if (diskRequired) { + if (valueOffset > fieldEnd - 4) { + throw new ZipException("Invalid ZIP64 extra field: " + path); + } + values.add(unsignedInt(extra, valueOffset)); + } + long[] result = new long[values.size()]; + for (int index = 0; index < result.length; index++) { + result[index] = values.get(index); + } + return result; + } + offset += size; + } + throw new ZipException("Required ZIP64 extra field not found: " + path); + } + + private static long littleEndianLong( + byte[] bytes, + int offset, + int fieldEnd, + Path path) throws ZipException { + if (offset > fieldEnd - 8 || offset > bytes.length - 8) { + throw new ZipException("Invalid ZIP64 extra field: " + path); + } + long low = unsignedInt(bytes, offset); + long high = unsignedInt(bytes, offset + 4); + long value = low | (high << 32); + if (value < 0L) { + throw new ZipException("ZIP64 value is too large: " + path); + } + return value; + } + + private static int unsignedShort(byte[] bytes, int offset) { + return Byte.toUnsignedInt(bytes[offset]) | (Byte.toUnsignedInt(bytes[offset + 1]) << 8); + } + + private static long unsignedInt(byte[] bytes, int offset) { + return (long) Byte.toUnsignedInt(bytes[offset]) + | ((long) Byte.toUnsignedInt(bytes[offset + 1]) << 8) + | ((long) Byte.toUnsignedInt(bytes[offset + 2]) << 16) + | ((long) Byte.toUnsignedInt(bytes[offset + 3]) << 24); + } + + private static long findEndOfCentralDirectory(RandomAccessFile file) throws IOException { + long minimumOffset = Math.max(0L, + file.length() - END_OF_CENTRAL_DIRECTORY_SIZE - MAXIMUM_COMMENT_LENGTH); + for (long offset = file.length() - END_OF_CENTRAL_DIRECTORY_SIZE; + offset >= minimumOffset; + offset--) { + file.seek(offset); + if (readUnsignedInt(file) == END_OF_CENTRAL_DIRECTORY) { + file.seek(offset + 20); + int commentLength = readUnsignedShort(file); + if (offset + END_OF_CENTRAL_DIRECTORY_SIZE + commentLength == file.length()) { + return offset; + } + } + } + throw new ZipException("ZIP end of central directory not found"); + } + + private static Directory readZip64Directory( + RandomAccessFile file, + Path path, + long endOffset) throws IOException { + long locatorOffset = endOffset - 20L; + if (locatorOffset < 0L) { + throw new ZipException("ZIP64 locator not found: " + path); + } + file.seek(locatorOffset); + if (readUnsignedInt(file) != ZIP64_END_OF_CENTRAL_DIRECTORY_LOCATOR) { + throw new ZipException("ZIP64 locator not found: " + path); + } + if (readUnsignedInt(file) != 0L) { + throw new ZipException("Split ZIP64 archives are not supported: " + path); + } + long zip64Offset = readLong(file); + if (readUnsignedInt(file) != 1L || zip64Offset < 0L || zip64Offset > file.length() - 56L) { + throw new ZipException("Invalid ZIP64 locator: " + path); + } + file.seek(zip64Offset); + if (readUnsignedInt(file) != ZIP64_END_OF_CENTRAL_DIRECTORY) { + throw new ZipException("ZIP64 end of central directory not found: " + path); + } + long recordSize = readLong(file); + if (recordSize < 44L) { + throw new ZipException("Invalid ZIP64 end of central directory: " + path); + } + skip(file, 4, file.length(), path); + long diskNumber = readUnsignedInt(file); + long centralDirectoryDisk = readUnsignedInt(file); + long entriesOnDisk = readLong(file); + long totalEntries = readLong(file); + long centralDirectorySize = readLong(file); + long centralDirectoryOffset = readLong(file); + if (diskNumber != 0L || centralDirectoryDisk != 0L || entriesOnDisk != totalEntries) { + throw new ZipException("Split ZIP64 archives are not supported: " + path); + } + return checkedDirectory( + path, + file.length(), + totalEntries, + centralDirectorySize, + centralDirectoryOffset); + } + + private static Directory checkedDirectory( + Path path, + long fileLength, + long entryCount, + long size, + long offset) throws ZipException { + if (entryCount < 0L || size < 0L || offset < 0L || offset > fileLength - size) { + throw new ZipException("Invalid ZIP central directory bounds: " + path); + } + return new Directory(entryCount, size, offset); + } + + private static int readUnsignedShort(RandomAccessFile file) throws IOException { + return file.readUnsignedByte() | (file.readUnsignedByte() << 8); + } + + private static long readUnsignedInt(RandomAccessFile file) throws IOException { + return (long) file.readUnsignedByte() + | ((long) file.readUnsignedByte() << 8) + | ((long) file.readUnsignedByte() << 16) + | ((long) file.readUnsignedByte() << 24); + } + + private static long readLong(RandomAccessFile file) throws IOException { + long value = readUnsignedInt(file) | (readUnsignedInt(file) << 32); + if (value < 0L) { + throw new ZipException("ZIP64 value is too large"); + } + return value; + } + + private static void skip( + RandomAccessFile file, + long count, + long maximumOffset, + Path path) throws IOException { + long current = file.getFilePointer(); + long target = current + count; + if (count < 0L || target < current || target > maximumOffset) { + throw new ZipException("Invalid ZIP central directory field length: " + path); + } + file.seek(target); + } + + private static LogFileReadLimitExceededException entryLimit( + Path path, + int maximumEntries, + long observedEntries) { + return new LogFileReadLimitExceededException( + LogFileReadLimitExceededException.LimitType.ARCHIVE_ENTRIES, + path, + null, + Integer.toString(maximumEntries), + Long.toString(observedEntries)); + } + + static final class Index { + private final long centralDirectoryOffset; + private final List entries; + + private Index(long centralDirectoryOffset, List entries) { + this.centralDirectoryOffset = centralDirectoryOffset; + this.entries = Collections.unmodifiableList(entries); + } + + long getCentralDirectoryOffset() { + return centralDirectoryOffset; + } + + List getEntries() { + return entries; + } + + Entry firstFile() { + return entries.stream().filter(entry -> !entry.isDirectory()).findFirst().orElse(null); + } + + Entry find(String name) { + return entries.stream().filter(entry -> entry.getName().equals(name)).findFirst().orElse(null); + } + } + + static final class Entry { + private final String name; + private final boolean directory; + private final int flags; + private final int method; + private final long crc; + private final long compressedSize; + private final long expandedSize; + private final long localHeaderOffset; + private final boolean zip64Size; + + private Entry( + String name, + boolean directory, + int flags, + int method, + long crc, + long compressedSize, + long expandedSize, + long localHeaderOffset, + boolean zip64Size) { + this.name = name; + this.directory = directory; + this.flags = flags; + this.method = method; + this.crc = crc; + this.compressedSize = compressedSize; + this.expandedSize = expandedSize; + this.localHeaderOffset = localHeaderOffset; + this.zip64Size = zip64Size; + } + + String getName() { + return name; + } + + boolean isDirectory() { + return directory; + } + + int getMethod() { + return method; + } + + int getFlags() { + return flags; + } + + long getCrc() { + return crc; + } + + long getCompressedSize() { + return compressedSize; + } + + long getExpandedSize() { + return expandedSize; + } + + long getLocalHeaderOffset() { + return localHeaderOffset; + } + + boolean isZip64Size() { + return zip64Size; + } + } + + private static final class Directory { + private final long entryCount; + private final long size; + private final long offset; + + private Directory(long entryCount, long size, long offset) { + this.entryCount = entryCount; + this.size = size; + this.offset = offset; + } + } +} diff --git a/api/src/main/java/com/microsoft/gctoolkit/jvm/AbstractJavaVirtualMachine.java b/api/src/main/java/com/microsoft/gctoolkit/jvm/AbstractJavaVirtualMachine.java index bd7db30f2..33de2fd2a 100644 --- a/api/src/main/java/com/microsoft/gctoolkit/jvm/AbstractJavaVirtualMachine.java +++ b/api/src/main/java/com/microsoft/gctoolkit/jvm/AbstractJavaVirtualMachine.java @@ -15,6 +15,7 @@ import com.microsoft.gctoolkit.time.DateTimeStamp; import java.io.IOException; +import java.io.UncheckedIOException; import java.util.List; import java.util.Map; import java.util.Optional; @@ -23,6 +24,7 @@ import java.util.concurrent.Phaser; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.stream.Stream; /** * The base implementation of JavaVirtualMachine that uses the message API to feed @@ -178,7 +180,9 @@ public void analyze(List> registeredAggregator try { if (finishLine.getRegisteredParties() > 0) { - dataSource.stream().forEach(message -> dataSourceBus.publish(ChannelName.DATA_SOURCE, message)); + try (Stream lines = dataSource.stream()) { + lines.forEach(message -> dataSourceBus.publish(ChannelName.DATA_SOURCE, message)); + } finishLine.awaitAdvance(0); } else { LOGGER.log(Level.INFO, "No Aggregations have been registered, DataSource will not be analysed."); @@ -194,7 +198,7 @@ public void analyze(List> registeredAggregator setEstimatedJVMStartTime(terminationRecord.estimatedStartTime()); }); } catch (IOException ioe) { - LOGGER.log(Level.SEVERE, ioe.getMessage(), ioe); + throw new UncheckedIOException(ioe); } finally { dataSourceBus.close(); eventBus.close(); diff --git a/api/src/test/java/com/microsoft/gctoolkit/io/SingleGCLogFileReadLimitsTest.java b/api/src/test/java/com/microsoft/gctoolkit/io/SingleGCLogFileReadLimitsTest.java new file mode 100644 index 000000000..165130873 --- /dev/null +++ b/api/src/test/java/com/microsoft/gctoolkit/io/SingleGCLogFileReadLimitsTest.java @@ -0,0 +1,812 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +package com.microsoft.gctoolkit.io; + +import com.microsoft.gctoolkit.GCToolKit; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.zip.CRC32; +import java.util.zip.GZIPOutputStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import static com.microsoft.gctoolkit.io.GCLogFile.END_OF_DATA_SENTINEL; +import static com.microsoft.gctoolkit.io.LogFileReadLimitExceededException.LimitType.ARCHIVE_ENTRIES; +import static com.microsoft.gctoolkit.io.LogFileReadLimitExceededException.LimitType.COMPRESSED_BYTES; +import static com.microsoft.gctoolkit.io.LogFileReadLimitExceededException.LimitType.COMPRESSED_MEMBERS; +import static com.microsoft.gctoolkit.io.LogFileReadLimitExceededException.LimitType.COMPRESSION_RATIO; +import static com.microsoft.gctoolkit.io.LogFileReadLimitExceededException.LimitType.EXPANDED_BYTES; +import static com.microsoft.gctoolkit.io.LogFileReadLimitExceededException.LimitType.HEADER_BYTES; +import static com.microsoft.gctoolkit.io.LogFileReadLimitExceededException.LimitType.LINE_CHARACTERS; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class SingleGCLogFileReadLimitsTest { + + private static final int DEFAULT_MAX_LINE_CHARACTERS = 1024 * 1024; + private static final int HIGHLY_COMPRESSIBLE_LINE_COUNT = 2048; + private static final String HIGHLY_COMPRESSIBLE_LINE = "a".repeat(1024); + private static final LogFileReadLimits SMALL_LIMITS = + new LogFileReadLimits(64 * 1024, 2048, 10_000.0d, 1024 * 1024, 16); + + @TempDir + Path temporaryDirectory; + + @Test + void rejectsLineLongerThanDefaultLimit() throws IOException { + Path log = temporaryDirectory.resolve("long-line.log"); + Files.writeString(log, "a".repeat(DEFAULT_MAX_LINE_CHARACTERS + 1), StandardCharsets.UTF_8); + + LogFileReadLimitExceededException failure = assertThrows(LogFileReadLimitExceededException.class, () -> { + try (Stream stream = new SingleGCLogFile(log).stream()) { + stream.findFirst(); + } + }); + assertEquals(LINE_CHARACTERS, failure.getLimitType()); + assertEquals(log, failure.getPath()); + } + + @Test + void rejectsExcessiveGzipCompressionRatio() throws IOException { + Path log = temporaryDirectory.resolve("high-ratio.log.gz"); + try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter( + new GZIPOutputStream(Files.newOutputStream(log)), StandardCharsets.UTF_8))) { + for (int index = 0; index < HIGHLY_COMPRESSIBLE_LINE_COUNT; index++) { + writer.write(HIGHLY_COMPRESSIBLE_LINE); + writer.newLine(); + } + } + + LogFileReadLimitExceededException failure = assertThrows(LogFileReadLimitExceededException.class, () -> { + try (Stream stream = new SingleGCLogFile(log).stream()) { + stream.count(); + } + }); + assertEquals(COMPRESSION_RATIO, failure.getLimitType()); + } + + @Test + void acceptsLineAtConfiguredLimitAndCommonLineEndings() throws IOException { + Path log = temporaryDirectory.resolve("line-endings.log"); + Files.writeString(log, "ab\r\ncd\ref\néé", StandardCharsets.UTF_8); + LogFileReadLimits limits = new LogFileReadLimits(128, 2, 100.0d, 64, 16); + + try (Stream stream = new SingleGCLogFile(log, limits).stream()) { + assertEquals( + List.of("ab", "cd", "ef", "éé", END_OF_DATA_SENTINEL), + stream.collect(Collectors.toList())); + } + } + + @Test + void rejectsPlaintextExpandedBytesOverConfiguredLimit() throws IOException { + Path log = temporaryDirectory.resolve("expanded.log"); + Files.writeString(log, repeatedLines(80, 1023), StandardCharsets.UTF_8); + + assertLimit(EXPANDED_BYTES, () -> { + try (Stream stream = new SingleGCLogFile(log, SMALL_LIMITS).stream()) { + stream.count(); + } + }); + } + + @Test + void rejectsGzipExpandedBytesOverConfiguredLimitAcrossMembers() throws IOException { + Path log = temporaryDirectory.resolve("concatenated.log.gz"); + writeGzipMember(log, repeatedLines(40, 1023), false); + writeGzipMember(log, repeatedLines(40, 1023), true); + + assertLimit(EXPANDED_BYTES, () -> { + try (Stream stream = new SingleGCLogFile(log, SMALL_LIMITS).stream()) { + stream.count(); + } + }); + } + + @Test + void acceptsExpandedBytesAtConfiguredLimit() throws IOException { + Path log = temporaryDirectory.resolve("expanded-boundary.log"); + Files.writeString(log, repeatedLines(64, 1023), StandardCharsets.UTF_8); + + try (Stream stream = new SingleGCLogFile(log, SMALL_LIMITS).stream()) { + assertEquals(65L, stream.count()); + } + } + + @Test + void rejectsZipExpandedBytesOverConfiguredLimit() throws IOException { + Path log = temporaryDirectory.resolve("expanded.log.zip"); + writeZip(log, Map.of("gc.log", repeatedLines(80, 1023))); + + assertLimit(EXPANDED_BYTES, () -> { + try (Stream stream = new SingleGCLogFile(log, SMALL_LIMITS).stream()) { + stream.count(); + } + }); + } + + @Test + void rejectsLongLineInGzip() throws IOException { + Path log = temporaryDirectory.resolve("long-line.log.gz"); + writeGzipMember(log, "a".repeat(2049), false); + + assertLimit(LINE_CHARACTERS, () -> { + try (Stream stream = new SingleGCLogFile(log, SMALL_LIMITS).stream()) { + stream.findFirst(); + } + }); + } + + @Test + void rejectsLongLineInZip() throws IOException { + Path log = temporaryDirectory.resolve("long-line.log.zip"); + writeZip(log, Map.of("gc.log", "a".repeat(2049))); + + assertLimit(LINE_CHARACTERS, () -> { + try (Stream stream = new SingleGCLogFile(log, SMALL_LIMITS).stream()) { + stream.findFirst(); + } + }); + } + + @Test + void rejectsExcessiveZipCompressionRatio() throws IOException { + Path log = temporaryDirectory.resolve("high-ratio.log.zip"); + writeZip(log, Map.of("gc.log", repeatedLines(2048, 1023))); + + assertLimit(COMPRESSION_RATIO, () -> { + try (Stream stream = new SingleGCLogFile(log).stream()) { + stream.count(); + } + }); + } + + @Test + void acceptsCompressionRatioWithinGraceThreshold() throws IOException { + Path log = temporaryDirectory.resolve("ratio-grace.log.gz"); + writeGzipMember(log, "line\n", false); + LogFileReadLimits limits = new LogFileReadLimits(1024, 128, 0.1d, 128, 16); + + try (Stream stream = new SingleGCLogFile(log, limits).stream()) { + assertEquals(2L, stream.count()); + } + } + + @Test + void rejectsHighRatioConcatenatedGzipMemberAfterLowRatioMember() throws IOException { + Path log = temporaryDirectory.resolve("mixed-ratio-members.log.gz"); + writeGzipMember(log, randomAsciiLine(50_000), false); + writeGzipMember(log, "a".repeat(100_000) + "\n", true); + LogFileReadLimits limits = new LogFileReadLimits(256 * 1024, 200_000, 10.0d, 1024, 16); + + assertLimit(COMPRESSION_RATIO, () -> { + try (Stream stream = new SingleGCLogFile(log, limits).stream()) { + stream.count(); + } + }); + } + + @Test + void appliesCompressionRatioGraceOnceAcrossGzipMembers() throws IOException { + Path log = temporaryDirectory.resolve("split-grace-members.log.gz"); + for (int index = 0; index < 10; index++) { + writeGzipMember(log, "a".repeat(1024), index != 0); + } + LogFileReadLimits limits = new LogFileReadLimits(32 * 1024, 16 * 1024, 2.0d, 1024, 16); + + assertLimit(COMPRESSION_RATIO, () -> { + try (Stream stream = new SingleGCLogFile(log, limits).stream()) { + stream.count(); + } + }); + } + + @Test + void appliesCompressionRatioGraceOnceAcrossZipEntries() throws IOException { + Path log = temporaryDirectory.resolve("split-grace-entries.zip"); + writeZip(log, Map.of( + "gc.log.0", "[1.0s][info][gc] " + "a".repeat(800) + "\n", + "gc.log.1", "[2.0s][info][gc] " + "a".repeat(800) + "\n", + "gc.log", "[3.0s][info][gc] " + "a".repeat(800) + "\n")); + LogFileReadLimits limits = new LogFileReadLimits(32 * 1024, 2048, 2.0d, 1024, 16); + + assertLimit( + COMPRESSION_RATIO, + () -> new RotatingGCLogFile(log, limits).getMetaData().getNumberOfFiles()); + } + + @Test + void validatesGzipHeaderCrc() throws IOException { + Path valid = temporaryDirectory.resolve("valid-header-crc.log.gz"); + writeGzipWithHeaderCrc(valid, "line\n", false); + try (Stream stream = new SingleGCLogFile(valid, SMALL_LIMITS).stream()) { + assertEquals(2L, stream.count()); + } + + Path invalid = temporaryDirectory.resolve("invalid-header-crc.log.gz"); + writeGzipWithHeaderCrc(invalid, "line\n", true); + assertThrows(UncheckedIOException.class, () -> { + try (Stream stream = new SingleGCLogFile(invalid, SMALL_LIMITS).stream()) { + stream.count(); + } + }); + } + + @Test + void rejectsExcessiveGzipMembersEvenWhenEmpty() throws IOException { + Path log = temporaryDirectory.resolve("empty-members.log.gz"); + for (int index = 0; index < 5; index++) { + writeGzipMember(log, "", index != 0); + } + LogFileReadLimits limits = + new LogFileReadLimits(64 * 1024, 64 * 1024, 1024, 100.0d, 1024, 4, 1024); + + assertLimit(COMPRESSED_MEMBERS, () -> { + try (Stream stream = new SingleGCLogFile(log, limits).stream()) { + stream.count(); + } + }); + } + + @Test + void rejectsOversizedGzipHeaderField() throws IOException { + Path log = temporaryDirectory.resolve("long-header.log.gz"); + writeGzipWithFileName(log, "a".repeat(64), "line\n"); + LogFileReadLimits limits = + new LogFileReadLimits(64 * 1024, 64 * 1024, 1024, 100.0d, 1024, 16, 32); + + assertLimit(HEADER_BYTES, () -> { + try (Stream stream = new SingleGCLogFile(log, limits).stream()) { + stream.count(); + } + }); + } + + @Test + void rejectsOversizedCompressedGzipInput() throws IOException { + Path log = temporaryDirectory.resolve("compressed-input.log.gz"); + writeGzipMember(log, randomAsciiLine(4096), false); + LogFileReadLimits limits = + new LogFileReadLimits(512, 16 * 1024, 16 * 1024, 100.0d, 1024, 16, 1024); + + assertLimit(COMPRESSED_BYTES, () -> { + try (Stream stream = new SingleGCLogFile(log, limits).stream()) { + stream.count(); + } + }); + } + + @Test + void ignoresPayloadOfZipDirectoryEntry() throws IOException { + Path log = temporaryDirectory.resolve("directory-payload.zip"); + try (ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(log))) { + zip.putNextEntry(new ZipEntry("ignored/")); + zip.write("a".repeat(128 * 1024).getBytes(StandardCharsets.UTF_8)); + zip.closeEntry(); + zip.putNextEntry(new ZipEntry("gc.log")); + zip.write("line\n".getBytes(StandardCharsets.UTF_8)); + zip.closeEntry(); + } + LogFileReadLimits limits = new LogFileReadLimits(1024, 128, 2.0d, 128, 16); + + try (Stream stream = new SingleGCLogFile(log, limits).stream()) { + assertEquals(2L, stream.count()); + } + } + + @Test + void readsFirstFileAfterZipDirectory() throws IOException { + Path log = temporaryDirectory.resolve("directory-first.zip"); + try (ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(log))) { + zip.putNextEntry(new ZipEntry("logs/")); + zip.closeEntry(); + zip.putNextEntry(new ZipEntry("logs/gc.log")); + zip.write("line\n".getBytes(StandardCharsets.UTF_8)); + zip.closeEntry(); + } + + try (Stream stream = new SingleGCLogFile(log, SMALL_LIMITS).stream()) { + assertEquals(List.of("line", END_OF_DATA_SENTINEL), stream.collect(Collectors.toList())); + } + } + + @Test + void readsAndValidatesStoredZipEntry() throws IOException { + Path log = temporaryDirectory.resolve("stored.zip"); + byte[] content = "line\n".getBytes(StandardCharsets.UTF_8); + CRC32 crc = new CRC32(); + crc.update(content); + try (ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(log))) { + ZipEntry entry = new ZipEntry("gc.log"); + entry.setMethod(ZipEntry.STORED); + entry.setSize(content.length); + entry.setCompressedSize(content.length); + entry.setCrc(crc.getValue()); + zip.putNextEntry(entry); + zip.write(content); + zip.closeEntry(); + } + + try (Stream stream = new SingleGCLogFile(log, SMALL_LIMITS).stream()) { + assertEquals(2L, stream.count()); + } + } + + @Test + void readsStoredZipEntryWithDataDescriptor() throws IOException { + Path log = temporaryDirectory.resolve("stored-descriptor.zip"); + byte[] content = "line\n".getBytes(StandardCharsets.UTF_8); + CRC32 crc = new CRC32(); + crc.update(content); + try (ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(log))) { + ZipEntry entry = new ZipEntry("gc.log"); + entry.setMethod(ZipEntry.STORED); + entry.setSize(content.length); + entry.setCompressedSize(content.length); + entry.setCrc(crc.getValue()); + zip.putNextEntry(entry); + zip.write(content); + zip.closeEntry(); + } + Files.write(log, withStoredDataDescriptor(Files.readAllBytes(log))); + + try (Stream stream = new SingleGCLogFile(log, SMALL_LIMITS).stream()) { + assertEquals(2L, stream.count()); + } + } + + @Test + void rejectsEmptyZip() throws IOException { + Path log = temporaryDirectory.resolve("empty.zip"); + try (ZipOutputStream ignored = new ZipOutputStream(Files.newOutputStream(log))) { + // Empty archive. + } + + assertThrows(IOException.class, () -> new SingleGCLogFile(log, SMALL_LIMITS).stream()); + } + + @Test + void rejectsZipWithStaleCentralDirectoryCrc() throws IOException { + Path log = temporaryDirectory.resolve("stale-central-crc.zip"); + writeZip(log, Map.of("gc.log", "line\n")); + byte[] archive = Files.readAllBytes(log); + int centralDirectory = findSignature(archive, 0x02014b50); + archive[centralDirectory + 16] ^= 1; + Files.write(log, archive); + + assertThrows(UncheckedIOException.class, () -> { + try (Stream stream = new SingleGCLogFile(log, SMALL_LIMITS).stream()) { + stream.count(); + } + }); + } + + @Test + void rejectsZipWithForgedCentralDirectoryCompressedSize() throws IOException { + Path log = temporaryDirectory.resolve("forged-compressed-size.zip"); + writeZip(log, Map.of("gc.log", "a".repeat(8192) + "\n")); + byte[] archive = Files.readAllBytes(log); + int centralDirectory = findSignature(archive, 0x02014b50); + putUnsignedInt(archive, centralDirectory + 20, 1024 * 1024); + Files.write(log, archive); + LogFileReadLimits limits = new LogFileReadLimits(16 * 1024, 16 * 1024, 10_000.0d, 1024, 16); + + assertThrows(UncheckedIOException.class, () -> { + try (Stream stream = new SingleGCLogFile(log, limits).stream()) { + stream.count(); + } + }); + } + + @Test + void rejectsOversizedZip64DeclaredEntryCount() throws IOException { + Path log = temporaryDirectory.resolve("forged-zip64-count.zip"); + writeZip(log, Map.of("gc.log", "line\n")); + Files.write(log, withZip64EntryCount(Files.readAllBytes(log), 500_000_000L)); + + LogFileReadLimitExceededException failure = assertThrows( + LogFileReadLimitExceededException.class, + () -> new SingleGCLogFile(log, SMALL_LIMITS).stream()); + assertEquals(ARCHIVE_ENTRIES, failure.getLimitType()); + } + + @Test + void rejectsOversizedCompressedZipInput() throws IOException { + Path log = temporaryDirectory.resolve("compressed-input.zip"); + writeZip(log, Map.of("gc.log", randomAsciiLine(4096))); + LogFileReadLimits limits = + new LogFileReadLimits(512, 16 * 1024, 16 * 1024, 100.0d, 1024, 16, 1024); + + assertLimit(COMPRESSED_BYTES, () -> new SingleGCLogFile(log, limits).stream()); + } + + @Test + void rejectsExcessiveZipControlRecords() throws IOException { + Path log = temporaryDirectory.resolve("control-records.zip"); + Files.write(log, zipWithArchiveExtraRecords(5)); + LogFileReadLimits limits = + new LogFileReadLimits(64 * 1024, 64 * 1024, 1024, 100.0d, 1024, 2, 1024); + + assertLimit(ARCHIVE_ENTRIES, () -> new SingleGCLogFile(log, limits).stream()); + } + + @Test + void rejectsExcessiveDirectorySegments() throws IOException { + Path directory = temporaryDirectory.resolve("rotating"); + Files.createDirectory(directory); + Files.writeString(directory.resolve("gc.log"), "[3.0s][info][gc] current\n"); + Files.writeString(directory.resolve("gc.log.0"), "[1.0s][info][gc] first\n"); + Files.writeString(directory.resolve("gc.log.1"), "[2.0s][info][gc] second\n"); + LogFileReadLimits limits = + new LogFileReadLimits(64 * 1024, 64 * 1024, 1024, 100.0d, 1024, 2, 1024); + + assertLimit( + ARCHIVE_ENTRIES, + () -> new RotatingGCLogFile(directory, limits).getMetaData().getNumberOfFiles()); + } + + @Test + void doesNotEmitSentinelAfterLimitFailure() throws IOException { + Path log = temporaryDirectory.resolve("sentinel.log"); + Files.writeString(log, "ok\n" + "a".repeat(2049), StandardCharsets.UTF_8); + List consumed = new ArrayList<>(); + + assertLimit(LINE_CHARACTERS, () -> { + try (Stream stream = new SingleGCLogFile(log, SMALL_LIMITS).stream()) { + stream.forEach(consumed::add); + } + }); + assertEquals(List.of("ok"), consumed); + assertFalse(consumed.contains(END_OF_DATA_SENTINEL)); + } + + @Test + void closesFileAfterLimitFailure() throws IOException { + Path log = temporaryDirectory.resolve("closed-after-failure.log"); + Files.writeString(log, "a".repeat(2049), StandardCharsets.UTF_8); + + assertLimit(LINE_CHARACTERS, () -> { + try (Stream stream = new SingleGCLogFile(log, SMALL_LIMITS).stream()) { + stream.findFirst(); + } + }); + assertDoesNotThrow(() -> Files.delete(log)); + assertFalse(Files.exists(log)); + } + + @Test + void analysisPreservesLimitFailure() throws IOException { + Path log = temporaryDirectory.resolve("analysis-limit.log"); + Files.writeString(log, "a".repeat(2049), StandardCharsets.UTF_8); + + assertLimit( + LINE_CHARACTERS, + () -> new GCToolKit().analyze(new SingleGCLogFile(log, SMALL_LIMITS))); + } + + @Test + void rejectsAggregateExpandedBytesAcrossRotatingZipEntries() throws IOException { + Path log = temporaryDirectory.resolve("gc.log.zip"); + writeZip(log, Map.of( + "gc.log.0", timestampedLines(1.0d, 40, 1000), + "gc.log", timestampedLines(2.0d, 40, 1000))); + + assertLimit(EXPANDED_BYTES, () -> { + try (Stream stream = new RotatingGCLogFile(log, SMALL_LIMITS).stream()) { + stream.count(); + } + }); + } + + @Test + void rejectsExcessiveRotatingZipEntries() throws IOException { + Path log = temporaryDirectory.resolve("many-entries.zip"); + writeZip(log, Map.of( + "gc.log.0", "[1.0s][info][gc] first\n", + "gc.log.1", "[2.0s][info][gc] second\n", + "gc.log", "[3.0s][info][gc] current\n")); + LogFileReadLimits limits = new LogFileReadLimits(64 * 1024, 2048, 100.0d, 1024, 2); + + LogFileReadLimitExceededException failure = assertThrows( + LogFileReadLimitExceededException.class, + () -> new RotatingGCLogFile(log, limits).getMetaData().getNumberOfFiles()); + assertEquals(ARCHIVE_ENTRIES, failure.getLimitType()); + } + + @Test + void validatesLimitConfiguration() { + assertThrows(IllegalArgumentException.class, + () -> new LogFileReadLimits(0, 1, 1.0d, 1, 1)); + assertThrows(IllegalArgumentException.class, + () -> new LogFileReadLimits(1, 0, 1.0d, 1, 1)); + assertThrows(IllegalArgumentException.class, + () -> new LogFileReadLimits(1, 1, Double.POSITIVE_INFINITY, 1, 1)); + assertThrows(IllegalArgumentException.class, + () -> new LogFileReadLimits(1, 1, 1.0d, 0, 1)); + assertThrows(IllegalArgumentException.class, + () -> new LogFileReadLimits(1, 1, 1.0d, 1, 0)); + assertThrows(IllegalArgumentException.class, + () -> new LogFileReadLimits(0, 1, 1, 1.0d, 1, 1, 1)); + assertThrows(IllegalArgumentException.class, + () -> new LogFileReadLimits(1, 1, 1, 1.0d, 1, 1, 0)); + } + + @Test + void exposesFiniteSecureDefaults() { + LogFileReadLimits defaults = LogFileReadLimits.defaults(); + + assertTrue(defaults.getMaxCompressedBytes() > 0); + assertTrue(defaults.getMaxExpandedBytes() > 0); + assertTrue(defaults.getMaxLineCharacters() > 0); + assertTrue(Double.isFinite(defaults.getMaxCompressionRatio())); + assertTrue(defaults.getCompressionRatioGraceBytes() > 0); + assertTrue(defaults.getMaxArchiveEntries() > 0); + assertTrue(defaults.getMaxGzipHeaderBytes() > 0); + } + + @Test + void enforcesExpandedBudgetAtomically() throws Exception { + LogFileReadBudget budget = new LogFileReadBudget(1000); + ExecutorService executor = Executors.newFixedThreadPool(4); + CountDownLatch start = new CountDownLatch(1); + try { + List> results = new ArrayList<>(); + for (int index = 0; index < 4; index++) { + results.add(executor.submit(() -> { + start.await(); + try { + budget.record(1000, temporaryDirectory.resolve("parallel.log"), null); + return true; + } catch (LogFileReadLimitExceededException expected) { + return false; + } + })); + } + start.countDown(); + long successes = 0L; + for (Future result : results) { + if (result.get()) { + successes++; + } + } + assertEquals(1L, successes); + } finally { + executor.shutdownNow(); + } + } + + private static void assertLimit( + LogFileReadLimitExceededException.LimitType expected, + ThrowingOperation operation) { + LogFileReadLimitExceededException failure = + assertThrows(LogFileReadLimitExceededException.class, operation::run); + assertEquals(expected, failure.getLimitType()); + } + + private static String repeatedLines(int lineCount, int lineLength) { + return ("a".repeat(lineLength) + "\n").repeat(lineCount); + } + + private static String timestampedLines(double timestamp, int lineCount, int lineLength) { + StringBuilder content = new StringBuilder(lineCount * (lineLength + 32)); + for (int index = 0; index < lineCount; index++) { + content.append('[') + .append(timestamp + (index / 1000.0d)) + .append("s][info][gc] ") + .append("a".repeat(lineLength)) + .append('\n'); + } + return content.toString(); + } + + private static String randomAsciiLine(int length) { + Random random = new Random(123456789L); + StringBuilder line = new StringBuilder(length + 1); + for (int index = 0; index < length; index++) { + line.append((char) ('!' + random.nextInt('~' - '!' + 1))); + } + return line.append('\n').toString(); + } + + private static void writeGzipMember(Path path, String content, boolean append) throws IOException { + StandardOpenOption[] options = append + ? new StandardOpenOption[] {StandardOpenOption.CREATE, StandardOpenOption.APPEND} + : new StandardOpenOption[] {StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING}; + try (GZIPOutputStream gzip = new GZIPOutputStream(Files.newOutputStream(path, options))) { + gzip.write(content.getBytes(StandardCharsets.UTF_8)); + } + } + + private static void writeGzipWithHeaderCrc( + Path path, + String content, + boolean corruptChecksum) throws IOException { + Path ordinaryGzip = path.resolveSibling(path.getFileName() + ".ordinary"); + writeGzipMember(ordinaryGzip, content, false); + byte[] ordinary = Files.readAllBytes(ordinaryGzip); + ordinary[3] |= 2; + CRC32 headerCrc = new CRC32(); + headerCrc.update(ordinary, 0, 10); + int checksum = (int) headerCrc.getValue() & 0xffff; + if (corruptChecksum) { + checksum ^= 1; + } + byte[] withHeaderCrc = new byte[ordinary.length + 2]; + System.arraycopy(ordinary, 0, withHeaderCrc, 0, 10); + withHeaderCrc[10] = (byte) checksum; + withHeaderCrc[11] = (byte) (checksum >>> 8); + System.arraycopy(ordinary, 10, withHeaderCrc, 12, ordinary.length - 10); + Files.write(path, withHeaderCrc); + Files.delete(ordinaryGzip); + } + + private static void writeGzipWithFileName( + Path path, + String fileName, + String content) throws IOException { + Path ordinaryGzip = path.resolveSibling(path.getFileName() + ".ordinary"); + writeGzipMember(ordinaryGzip, content, false); + byte[] ordinary = Files.readAllBytes(ordinaryGzip); + ordinary[3] |= 8; + byte[] name = fileName.getBytes(StandardCharsets.ISO_8859_1); + byte[] withName = new byte[ordinary.length + name.length + 1]; + System.arraycopy(ordinary, 0, withName, 0, 10); + System.arraycopy(name, 0, withName, 10, name.length); + System.arraycopy(ordinary, 10, withName, 11 + name.length, ordinary.length - 10); + Files.write(path, withName); + Files.delete(ordinaryGzip); + } + + private static void writeZip(Path path, Map entries) throws IOException { + try (ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(path))) { + for (Map.Entry entry : entries.entrySet()) { + zip.putNextEntry(new ZipEntry(entry.getKey())); + zip.write(entry.getValue().getBytes(StandardCharsets.UTF_8)); + zip.closeEntry(); + } + } + } + + private static byte[] withZip64EntryCount(byte[] ordinaryZip, long entryCount) { + int end = findSignature(ordinaryZip, 0x06054b50); + long directorySize = unsignedInt(ordinaryZip, end + 12); + long directoryOffset = unsignedInt(ordinaryZip, end + 16); + byte[] result = new byte[ordinaryZip.length + 76]; + System.arraycopy(ordinaryZip, 0, result, 0, end); + + int zip64End = end; + putUnsignedInt(result, zip64End, 0x06064b50L); + putLong(result, zip64End + 4, 44L); + putUnsignedShort(result, zip64End + 12, 45); + putUnsignedShort(result, zip64End + 14, 45); + putUnsignedInt(result, zip64End + 16, 0L); + putUnsignedInt(result, zip64End + 20, 0L); + putLong(result, zip64End + 24, entryCount); + putLong(result, zip64End + 32, entryCount); + putLong(result, zip64End + 40, directorySize); + putLong(result, zip64End + 48, directoryOffset); + + int locator = zip64End + 56; + putUnsignedInt(result, locator, 0x07064b50L); + putUnsignedInt(result, locator + 4, 0L); + putLong(result, locator + 8, zip64End); + putUnsignedInt(result, locator + 16, 1L); + + int newEnd = locator + 20; + System.arraycopy(ordinaryZip, end, result, newEnd, ordinaryZip.length - end); + putUnsignedShort(result, newEnd + 8, 0xffff); + putUnsignedShort(result, newEnd + 10, 0xffff); + return result; + } + + private static byte[] withStoredDataDescriptor(byte[] ordinaryZip) { + int centralDirectory = findSignature(ordinaryZip, 0x02014b50); + int end = findSignature(ordinaryZip, 0x06054b50); + int localHeader = (int) unsignedInt(ordinaryZip, centralDirectory + 42); + int nameLength = Byte.toUnsignedInt(ordinaryZip[localHeader + 26]) + | (Byte.toUnsignedInt(ordinaryZip[localHeader + 27]) << 8); + int extraLength = Byte.toUnsignedInt(ordinaryZip[localHeader + 28]) + | (Byte.toUnsignedInt(ordinaryZip[localHeader + 29]) << 8); + int dataStart = localHeader + 30 + nameLength + extraLength; + int compressedSize = (int) unsignedInt(ordinaryZip, centralDirectory + 20); + int dataEnd = dataStart + compressedSize; + long crc = unsignedInt(ordinaryZip, centralDirectory + 16); + long expandedSize = unsignedInt(ordinaryZip, centralDirectory + 24); + + byte[] result = new byte[ordinaryZip.length + 16]; + System.arraycopy(ordinaryZip, 0, result, 0, dataEnd); + putUnsignedInt(result, dataEnd, 0x08074b50L); + putUnsignedInt(result, dataEnd + 4, crc); + putUnsignedInt(result, dataEnd + 8, compressedSize); + putUnsignedInt(result, dataEnd + 12, expandedSize); + System.arraycopy( + ordinaryZip, + dataEnd, + result, + dataEnd + 16, + ordinaryZip.length - dataEnd); + + result[localHeader + 6] |= 8; + for (int offset = 14; offset < 26; offset++) { + result[localHeader + offset] = 0; + } + int shiftedCentralDirectory = centralDirectory + 16; + result[shiftedCentralDirectory + 8] |= 8; + int shiftedEnd = end + 16; + putUnsignedInt(result, shiftedEnd + 16, shiftedCentralDirectory); + return result; + } + + private static byte[] zipWithArchiveExtraRecords(int recordCount) { + int directorySize = recordCount * 8; + byte[] archive = new byte[directorySize + 22]; + for (int index = 0; index < recordCount; index++) { + putUnsignedInt(archive, index * 8, 0x08064b50L); + putUnsignedInt(archive, index * 8 + 4, 0L); + } + putUnsignedInt(archive, directorySize, 0x06054b50L); + putUnsignedInt(archive, directorySize + 12, directorySize); + putUnsignedInt(archive, directorySize + 16, 0L); + return archive; + } + + private static int findSignature(byte[] bytes, int signature) { + for (int index = 0; index <= bytes.length - 4; index++) { + if ((int) unsignedInt(bytes, index) == signature) { + return index; + } + } + throw new IllegalArgumentException("ZIP signature not found"); + } + + private static long unsignedInt(byte[] bytes, int offset) { + return (long) Byte.toUnsignedInt(bytes[offset]) + | ((long) Byte.toUnsignedInt(bytes[offset + 1]) << 8) + | ((long) Byte.toUnsignedInt(bytes[offset + 2]) << 16) + | ((long) Byte.toUnsignedInt(bytes[offset + 3]) << 24); + } + + private static void putUnsignedShort(byte[] bytes, int offset, int value) { + bytes[offset] = (byte) value; + bytes[offset + 1] = (byte) (value >>> 8); + } + + private static void putUnsignedInt(byte[] bytes, int offset, long value) { + bytes[offset] = (byte) value; + bytes[offset + 1] = (byte) (value >>> 8); + bytes[offset + 2] = (byte) (value >>> 16); + bytes[offset + 3] = (byte) (value >>> 24); + } + + private static void putLong(byte[] bytes, int offset, long value) { + putUnsignedInt(bytes, offset, value); + putUnsignedInt(bytes, offset + 4, value >>> 32); + } + + @FunctionalInterface + private interface ThrowingOperation { + void run() throws Exception; + } +} From fc7a92a9655b7dfb6f3a645aac400f7a3f79f609 Mon Sep 17 00:00:00 2001 From: Derek Keeler Date: Wed, 26 Aug 2026 10:57:20 -0700 Subject: [PATCH 2/3] refactor(api): use Commons Compress for ZIP handling Delegate ZIP64 metadata and entry decoding to Commons Compress while retaining bounded preflight, resource accounting, and integrity checks. Keep the guarded GZIP reader for bounded optional-header parsing. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- api/pom.xml | 4 + .../gctoolkit/io/ArchiveSliceInputStream.java | 65 --- .../gctoolkit/io/CountingInputStream.java | 78 --- .../gctoolkit/io/GCLogFileZipSegment.java | 12 +- .../gctoolkit/io/LimitedGZIPInputStream.java | 78 ++- .../io/LogFileReadLimitExceededException.java | 1 + .../gctoolkit/io/LogFileReadLimits.java | 54 ++ .../gctoolkit/io/LogFileStreams.java | 321 +++++++++-- .../gctoolkit/io/RotatingLogFileMetadata.java | 7 +- .../io/ValidatedZipEntryInputStream.java | 277 ---------- .../gctoolkit/io/ZipArchivePreflight.java | 504 ++++++------------ api/src/main/java/module-info.java | 1 + .../io/SingleGCLogFileReadLimitsTest.java | 27 +- pom.xml | 6 + 14 files changed, 609 insertions(+), 826 deletions(-) delete mode 100644 api/src/main/java/com/microsoft/gctoolkit/io/ArchiveSliceInputStream.java delete mode 100644 api/src/main/java/com/microsoft/gctoolkit/io/CountingInputStream.java delete mode 100644 api/src/main/java/com/microsoft/gctoolkit/io/ValidatedZipEntryInputStream.java diff --git a/api/pom.xml b/api/pom.xml index 2836ee42e..1255f5d13 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -15,6 +15,10 @@ ${project.parent.url} + + org.apache.commons + commons-compress + org.junit.jupiter junit-jupiter-api diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/ArchiveSliceInputStream.java b/api/src/main/java/com/microsoft/gctoolkit/io/ArchiveSliceInputStream.java deleted file mode 100644 index 1f9deeb6e..000000000 --- a/api/src/main/java/com/microsoft/gctoolkit/io/ArchiveSliceInputStream.java +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. -package com.microsoft.gctoolkit.io; - -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Path; - -final class ArchiveSliceInputStream extends InputStream { - - private final FileInputStream inputStream; - private long remaining; - - ArchiveSliceInputStream(Path path, long offset, long length) throws IOException { - inputStream = new FileInputStream(path.toFile()); - try { - inputStream.getChannel().position(offset); - remaining = length; - } catch (IOException | RuntimeException | Error failure) { - try { - inputStream.close(); - } catch (IOException closeException) { - failure.addSuppressed(closeException); - } - throw failure; - } - } - - @Override - public int read() throws IOException { - if (remaining == 0L) { - return -1; - } - int value = inputStream.read(); - if (value != -1) { - remaining--; - } - return value; - } - - @Override - public int read(byte[] bytes, int offset, int length) throws IOException { - if (remaining == 0L) { - return -1; - } - int count = inputStream.read(bytes, offset, (int) Math.min(length, remaining)); - if (count > 0) { - remaining -= count; - } - return count; - } - - @Override - public long skip(long count) throws IOException { - long skipped = inputStream.skip(Math.min(count, remaining)); - remaining -= skipped; - return skipped; - } - - @Override - public void close() throws IOException { - inputStream.close(); - } -} diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/CountingInputStream.java b/api/src/main/java/com/microsoft/gctoolkit/io/CountingInputStream.java deleted file mode 100644 index de6feaf55..000000000 --- a/api/src/main/java/com/microsoft/gctoolkit/io/CountingInputStream.java +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. -package com.microsoft.gctoolkit.io; - -import java.io.FilterInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Path; - -final class CountingInputStream extends FilterInputStream { - - private final long maximumBytes; - private final Path path; - private long bytesRead; - - CountingInputStream(InputStream inputStream, long maximumBytes, Path path) { - super(inputStream); - this.maximumBytes = maximumBytes; - this.path = path; - } - - long getBytesRead() { - return bytesRead; - } - - @Override - public int read() throws IOException { - int value = super.read(); - if (value != -1) { - record(1L); - } - return value; - } - - @Override - public int read(byte[] bytes, int offset, int length) throws IOException { - int count = super.read(bytes, offset, nextReadLength(length)); - if (count > 0) { - record(count); - } - return count; - } - - @Override - public long skip(long count) throws IOException { - long skipped = super.skip(nextReadLength(count)); - record(skipped); - return skipped; - } - - private int nextReadLength(int requestedLength) { - return (int) nextReadLength((long) requestedLength); - } - - private long nextReadLength(long requestedLength) { - long remaining = maximumBytes - bytesRead; - long detectableLength = remaining == Long.MAX_VALUE ? Long.MAX_VALUE : remaining + 1L; - return Math.min(requestedLength, detectableLength); - } - - private void record(long count) { - if (count <= 0L) { - return; - } - if (count > maximumBytes - bytesRead) { - long observed = bytesRead == Long.MAX_VALUE - ? Long.MAX_VALUE - : bytesRead + count; - throw new LogFileReadLimitExceededException( - LogFileReadLimitExceededException.LimitType.COMPRESSED_BYTES, - path, - null, - Long.toString(maximumBytes), - Long.toString(observed)); - } - bytesRead += count; - } -} diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/GCLogFileZipSegment.java b/api/src/main/java/com/microsoft/gctoolkit/io/GCLogFileZipSegment.java index ee0fc1e21..fb2a4d8c0 100644 --- a/api/src/main/java/com/microsoft/gctoolkit/io/GCLogFileZipSegment.java +++ b/api/src/main/java/com/microsoft/gctoolkit/io/GCLogFileZipSegment.java @@ -28,8 +28,7 @@ public class GCLogFileZipSegment implements LogFileSegment { private final Path path; private final String segmentName; private final LogFileReadLimits readLimits; - private final ZipArchivePreflight.Index archiveIndex; - private final ZipArchivePreflight.Entry archiveEntry; + private final LogFileStreams.ZipEntryReference archiveEntry; private DateTimeStamp endTime = null; private DateTimeStamp startTime = null; @@ -50,19 +49,17 @@ public GCLogFileZipSegment(Path path, String segmentName) { * @param readLimits resource limits applied while streaming the segment */ public GCLogFileZipSegment(Path path, String segmentName, LogFileReadLimits readLimits) { - this(path, segmentName, readLimits, null, null); + this(path, segmentName, readLimits, null); } GCLogFileZipSegment( Path path, String segmentName, LogFileReadLimits readLimits, - ZipArchivePreflight.Index archiveIndex, - ZipArchivePreflight.Entry archiveEntry) { + LogFileStreams.ZipEntryReference archiveEntry) { this.path = path; this.segmentName = segmentName; this.readLimits = Objects.requireNonNull(readLimits, "readLimits"); - this.archiveIndex = archiveIndex; this.archiveEntry = archiveEntry; } @@ -168,10 +165,9 @@ public Stream stream() { Stream stream(LogFileReadBudget readBudget) { try { - if (archiveIndex != null && archiveEntry != null) { + if (archiveEntry != null) { return LogFileStreams.zipEntry( path, - archiveIndex, archiveEntry, readLimits, readBudget); diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/LimitedGZIPInputStream.java b/api/src/main/java/com/microsoft/gctoolkit/io/LimitedGZIPInputStream.java index 08cffde60..fb277646e 100644 --- a/api/src/main/java/com/microsoft/gctoolkit/io/LimitedGZIPInputStream.java +++ b/api/src/main/java/com/microsoft/gctoolkit/io/LimitedGZIPInputStream.java @@ -3,6 +3,7 @@ package com.microsoft.gctoolkit.io; import java.io.EOFException; +import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; import java.io.PushbackInputStream; @@ -21,7 +22,7 @@ final class LimitedGZIPInputStream extends InputStream { private static final int FLAG_COMMENT = 16; private static final int FLAG_RESERVED = 224; - private final CountingInputStream compressedSource; + private final CompressedCountingInputStream compressedSource; private final PushbackInputStream source; private final Inflater inflater = new Inflater(true); private final CRC32 crc = new CRC32(); @@ -45,7 +46,7 @@ final class LimitedGZIPInputStream extends InputStream { LogFileReadBudget budget, LogFileReadLimits limits, Path path) { - compressedSource = new CountingInputStream( + compressedSource = new CompressedCountingInputStream( inputStream, limits.getMaxCompressedBytes(), path); @@ -276,4 +277,77 @@ public void close() throws IOException { source.close(); } } + + private static final class CompressedCountingInputStream extends FilterInputStream { + + private final long maximumBytes; + private final Path path; + private long bytesRead; + + private CompressedCountingInputStream( + InputStream inputStream, + long maximumBytes, + Path path) { + super(inputStream); + this.maximumBytes = maximumBytes; + this.path = path; + } + + private long getBytesRead() { + return bytesRead; + } + + @Override + public int read() throws IOException { + int value = super.read(); + if (value != -1) { + record(1L); + } + return value; + } + + @Override + public int read(byte[] bytes, int offset, int length) throws IOException { + int count = super.read(bytes, offset, nextReadLength(length)); + if (count > 0) { + record(count); + } + return count; + } + + @Override + public long skip(long count) throws IOException { + long skipped = super.skip(nextReadLength(count)); + record(skipped); + return skipped; + } + + private int nextReadLength(int requestedLength) { + return (int) nextReadLength((long) requestedLength); + } + + private long nextReadLength(long requestedLength) { + long remaining = maximumBytes - bytesRead; + long detectableLength = remaining == Long.MAX_VALUE ? Long.MAX_VALUE : remaining + 1L; + return Math.min(requestedLength, detectableLength); + } + + private void record(long count) { + if (count <= 0L) { + return; + } + if (count > maximumBytes - bytesRead) { + long observed = bytesRead == Long.MAX_VALUE + ? Long.MAX_VALUE + : bytesRead + count; + throw new LogFileReadLimitExceededException( + LogFileReadLimitExceededException.LimitType.COMPRESSED_BYTES, + path, + null, + Long.toString(maximumBytes), + Long.toString(observed)); + } + bytesRead += count; + } + } } diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/LogFileReadLimitExceededException.java b/api/src/main/java/com/microsoft/gctoolkit/io/LogFileReadLimitExceededException.java index 29395623e..4e012b08d 100644 --- a/api/src/main/java/com/microsoft/gctoolkit/io/LogFileReadLimitExceededException.java +++ b/api/src/main/java/com/microsoft/gctoolkit/io/LogFileReadLimitExceededException.java @@ -21,6 +21,7 @@ public enum LimitType { LINE_CHARACTERS, COMPRESSION_RATIO, ARCHIVE_ENTRIES, + ARCHIVE_METADATA_BYTES, COMPRESSED_MEMBERS, HEADER_BYTES } diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/LogFileReadLimits.java b/api/src/main/java/com/microsoft/gctoolkit/io/LogFileReadLimits.java index ffb8e83cc..87122af53 100644 --- a/api/src/main/java/com/microsoft/gctoolkit/io/LogFileReadLimits.java +++ b/api/src/main/java/com/microsoft/gctoolkit/io/LogFileReadLimits.java @@ -30,6 +30,9 @@ public final class LogFileReadLimits { /** Default maximum number of file entries in a rotating ZIP log. */ public static final int DEFAULT_MAX_ARCHIVE_ENTRIES = 1024; + /** Default maximum number of bytes in a ZIP central directory. */ + public static final long DEFAULT_MAX_ARCHIVE_METADATA_BYTES = 16L * 1024L * 1024L; + /** Default maximum number of bytes in one GZIP member header. */ public static final int DEFAULT_MAX_GZIP_HEADER_BYTES = 64 * 1024; @@ -40,6 +43,7 @@ public final class LogFileReadLimits { DEFAULT_MAX_COMPRESSION_RATIO, DEFAULT_COMPRESSION_RATIO_GRACE_BYTES, DEFAULT_MAX_ARCHIVE_ENTRIES, + DEFAULT_MAX_ARCHIVE_METADATA_BYTES, DEFAULT_MAX_GZIP_HEADER_BYTES); private final long maxCompressedBytes; @@ -48,6 +52,7 @@ public final class LogFileReadLimits { private final double maxCompressionRatio; private final long compressionRatioGraceBytes; private final int maxArchiveEntries; + private final long maxArchiveMetadataBytes; private final int maxGzipHeaderBytes; /** @@ -72,6 +77,7 @@ public LogFileReadLimits( maxCompressionRatio, compressionRatioGraceBytes, maxArchiveEntries, + DEFAULT_MAX_ARCHIVE_METADATA_BYTES, DEFAULT_MAX_GZIP_HEADER_BYTES); } @@ -94,6 +100,38 @@ public LogFileReadLimits( long compressionRatioGraceBytes, int maxArchiveEntries, int maxGzipHeaderBytes) { + this( + maxCompressedBytes, + maxExpandedBytes, + maxLineCharacters, + maxCompressionRatio, + compressionRatioGraceBytes, + maxArchiveEntries, + DEFAULT_MAX_ARCHIVE_METADATA_BYTES, + maxGzipHeaderBytes); + } + + /** + * Creates a set of finite log-file read limits, including archive metadata limits. + * + * @param maxCompressedBytes maximum bytes read from one compressed input + * @param maxExpandedBytes maximum plaintext or expanded bytes per top-level stream operation + * @param maxLineCharacters maximum decoded characters in one line + * @param maxCompressionRatio maximum expanded-to-compressed byte ratio + * @param compressionRatioGraceBytes expanded bytes allowed before ratio enforcement begins + * @param maxArchiveEntries maximum ZIP entries, GZIP members, or rotating log segments + * @param maxArchiveMetadataBytes maximum bytes in a ZIP central directory + * @param maxGzipHeaderBytes maximum bytes in one GZIP member header + */ + public LogFileReadLimits( + long maxCompressedBytes, + long maxExpandedBytes, + int maxLineCharacters, + double maxCompressionRatio, + long compressionRatioGraceBytes, + int maxArchiveEntries, + long maxArchiveMetadataBytes, + int maxGzipHeaderBytes) { if (maxCompressedBytes <= 0) { throw new IllegalArgumentException("maxCompressedBytes must be positive"); } @@ -112,6 +150,9 @@ public LogFileReadLimits( if (maxArchiveEntries <= 0) { throw new IllegalArgumentException("maxArchiveEntries must be positive"); } + if (maxArchiveMetadataBytes <= 0) { + throw new IllegalArgumentException("maxArchiveMetadataBytes must be positive"); + } if (maxGzipHeaderBytes <= 0) { throw new IllegalArgumentException("maxGzipHeaderBytes must be positive"); } @@ -121,6 +162,7 @@ public LogFileReadLimits( this.maxCompressionRatio = maxCompressionRatio; this.compressionRatioGraceBytes = compressionRatioGraceBytes; this.maxArchiveEntries = maxArchiveEntries; + this.maxArchiveMetadataBytes = maxArchiveMetadataBytes; this.maxGzipHeaderBytes = maxGzipHeaderBytes; } @@ -187,6 +229,15 @@ public int getMaxArchiveEntries() { return maxArchiveEntries; } + /** + * Returns the maximum number of bytes accepted in a ZIP central directory. + * + * @return maximum ZIP central-directory bytes + */ + public long getMaxArchiveMetadataBytes() { + return maxArchiveMetadataBytes; + } + /** * Returns the maximum number of bytes accepted in one GZIP member header. * @@ -211,6 +262,7 @@ public boolean equals(Object other) { && Double.compare(maxCompressionRatio, that.maxCompressionRatio) == 0 && compressionRatioGraceBytes == that.compressionRatioGraceBytes && maxArchiveEntries == that.maxArchiveEntries + && maxArchiveMetadataBytes == that.maxArchiveMetadataBytes && maxGzipHeaderBytes == that.maxGzipHeaderBytes; } @@ -223,6 +275,7 @@ public int hashCode() { maxCompressionRatio, compressionRatioGraceBytes, maxArchiveEntries, + maxArchiveMetadataBytes, maxGzipHeaderBytes); } @@ -235,6 +288,7 @@ public String toString() { + ", maxCompressionRatio=" + maxCompressionRatio + ", compressionRatioGraceBytes=" + compressionRatioGraceBytes + ", maxArchiveEntries=" + maxArchiveEntries + + ", maxArchiveMetadataBytes=" + maxArchiveMetadataBytes + ", maxGzipHeaderBytes=" + maxGzipHeaderBytes + '}'; } diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/LogFileStreams.java b/api/src/main/java/com/microsoft/gctoolkit/io/LogFileStreams.java index 81cefe514..81c5c6f30 100644 --- a/api/src/main/java/com/microsoft/gctoolkit/io/LogFileStreams.java +++ b/api/src/main/java/com/microsoft/gctoolkit/io/LogFileStreams.java @@ -2,17 +2,30 @@ // Licensed under the MIT License. package com.microsoft.gctoolkit.io; +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipFile; +import org.apache.commons.compress.utils.InputStreamStatistics; + import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.UncheckedIOException; +import java.nio.channels.SeekableByteChannel; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.function.LongSupplier; import java.util.stream.Stream; import java.util.stream.StreamSupport; +import java.util.zip.CRC32; +import java.util.zip.ZipException; final class LogFileStreams { @@ -59,13 +72,14 @@ static Stream firstZipEntry( Path path, LogFileReadLimits limits, LogFileReadBudget budget) throws IOException { - ZipArchivePreflight.Index index = - ZipArchivePreflight.validate(path, limits); - ZipArchivePreflight.Entry entry = index.firstFile(); + ZipEntryReference entry = zipEntries(path, limits).stream() + .filter(candidate -> !candidate.isDirectory()) + .findFirst() + .orElse(null); if (entry == null) { throw new IOException("ZIP file contains no readable entries: " + path); } - return zipEntry(path, index, entry, limits, budget); + return zipEntry(path, entry, limits, budget); } static Stream zipEntry( @@ -73,13 +87,15 @@ static Stream zipEntry( String entryName, LogFileReadLimits limits, LogFileReadBudget budget) throws IOException { - ZipArchivePreflight.Index index = - ZipArchivePreflight.validate(path, limits); - ZipArchivePreflight.Entry entry = index.find(entryName); - if (entry == null || entry.isDirectory()) { + ZipEntryReference entry = zipEntries(path, limits).stream() + .filter(candidate -> !candidate.isDirectory()) + .filter(candidate -> candidate.getName().equals(entryName)) + .findFirst() + .orElse(null); + if (entry == null) { throw new IOException("ZIP entry not found: " + path + "!" + entryName); } - return zipEntry(path, index, entry, limits, budget); + return zipEntry(path, entry, limits, budget); } static Stream segment( @@ -128,34 +144,91 @@ private static Stream decodedLines( return StreamSupport.stream(lines, false).onClose(lines::closeUnchecked); } + static List zipEntries( + Path path, + LogFileReadLimits limits) throws IOException { + SeekableByteChannel channel = openValidatedZipChannel(path, limits); + ZipFile zipFile; + try { + zipFile = openZipFile(channel); + } catch (IOException exception) { + closeAfterFailure(channel, exception); + throw exception; + } + try (ZipFile closeableZipFile = zipFile) { + List references = new ArrayList<>(); + Map occurrences = new HashMap<>(); + Enumeration entries = closeableZipFile.getEntries(); + while (entries.hasMoreElements()) { + ZipArchiveEntry entry = entries.nextElement(); + if (references.size() == limits.getMaxArchiveEntries()) { + throw new LogFileReadLimitExceededException( + LogFileReadLimitExceededException.LimitType.ARCHIVE_ENTRIES, + path, + null, + Integer.toString(limits.getMaxArchiveEntries()), + Integer.toString(references.size() + 1)); + } + if (!entry.isDirectory()) { + validateEntryMetadata(closeableZipFile, entry, path); + } + int occurrence = occurrences.getOrDefault(entry.getName(), 0); + occurrences.put(entry.getName(), occurrence + 1); + references.add(new ZipEntryReference( + entry.getName(), + occurrence, + entry.isDirectory(), + entry.getMethod(), + entry.getCrc(), + entry.getCompressedSize(), + entry.getSize())); + } + return references; + } + } + static Stream zipEntry( Path path, - ZipArchivePreflight.Index index, - ZipArchivePreflight.Entry entry, + ZipEntryReference reference, LogFileReadLimits limits, LogFileReadBudget budget) throws IOException { - budget.rejectDeclaredSize(entry.getExpandedSize(), path, entry.getName()); - if (entry.getExpandedSize() > limits.getCompressionRatioGraceBytes()) { - double ratio = entry.getCompressedSize() == 0L - ? Double.POSITIVE_INFINITY - : (double) entry.getExpandedSize() / (double) entry.getCompressedSize(); - if (ratio > limits.getMaxCompressionRatio()) { - throw new LogFileReadLimitExceededException( - LogFileReadLimitExceededException.LimitType.COMPRESSION_RATIO, - path, - entry.getName(), - Double.toString(limits.getMaxCompressionRatio()), - Double.toString(ratio)); - } + SeekableByteChannel channel = openValidatedZipChannel(path, limits); + ZipFile zipFile; + try { + zipFile = openZipFile(channel); + } catch (IOException | RuntimeException | Error failure) { + closeAfterFailure(channel, failure); + throw failure; } - long archiveSliceLength = index.getCentralDirectoryOffset() - entry.getLocalHeaderOffset(); - ArchiveSliceInputStream archiveSlice = new ArchiveSliceInputStream( - path, - entry.getLocalHeaderOffset(), - archiveSliceLength); try { - ValidatedZipEntryInputStream validated = - new ValidatedZipEntryInputStream(archiveSlice, entry, path); + ZipArchiveEntry entry = resolveEntry(zipFile, reference, path); + validateEntryMetadata(zipFile, entry, path); + budget.rejectDeclaredSize(entry.getSize(), path, entry.getName()); + if (entry.getSize() > limits.getCompressionRatioGraceBytes()) { + double ratio = entry.getCompressedSize() == 0L + ? Double.POSITIVE_INFINITY + : (double) entry.getSize() / (double) entry.getCompressedSize(); + if (ratio > limits.getMaxCompressionRatio()) { + throw new LogFileReadLimitExceededException( + LogFileReadLimitExceededException.LimitType.COMPRESSION_RATIO, + path, + entry.getName(), + Double.toString(limits.getMaxCompressionRatio()), + Double.toString(ratio)); + } + } + + InputStream inputStream = zipFile.getInputStream(entry); + if (!(inputStream instanceof InputStreamStatistics)) { + throw new IOException("ZIP entry stream does not expose read statistics: " + + path + "!" + entry.getName()); + } + ValidatedZipInputStream validated = new ValidatedZipInputStream( + inputStream, + (InputStreamStatistics) inputStream, + zipFile, + entry, + path); return lines( validated, Charset.defaultCharset(), @@ -165,11 +238,60 @@ static Stream zipEntry( entry.getName(), validated::getCompressedBytesRead); } catch (IOException | RuntimeException | Error failure) { - closeAfterFailure(archiveSlice, failure); + closeAfterFailure(zipFile, failure); + throw failure; + } + } + + private static SeekableByteChannel openValidatedZipChannel( + Path path, + LogFileReadLimits limits) throws IOException { + SeekableByteChannel channel = Files.newByteChannel(path, StandardOpenOption.READ); + try { + ZipArchivePreflight.validate(channel, path, limits); + return channel; + } catch (IOException | RuntimeException | Error failure) { + closeAfterFailure(channel, failure); throw failure; } } + private static ZipFile openZipFile(SeekableByteChannel channel) throws IOException { + return ZipFile.builder().setSeekableByteChannel(channel).get(); + } + + private static ZipArchiveEntry resolveEntry( + ZipFile zipFile, + ZipEntryReference reference, + Path path) throws IOException { + int occurrence = 0; + for (ZipArchiveEntry entry : zipFile.getEntries(reference.getName())) { + if (occurrence++ == reference.occurrence) { + if (!reference.matches(entry)) { + throw new ZipException( + "ZIP entry metadata changed while reading: " + + path + "!" + reference.getName()); + } + return entry; + } + } + throw new IOException("ZIP entry not found: " + path + "!" + reference.getName()); + } + + private static void validateEntryMetadata( + ZipFile zipFile, + ZipArchiveEntry entry, + Path path) throws ZipException { + if (!zipFile.canReadEntryData(entry)) { + throw new ZipException( + "Unsupported or encrypted ZIP entry: " + path + "!" + entry.getName()); + } + if (entry.getSize() < 0L || entry.getCompressedSize() < 0L || entry.getCrc() < 0L) { + throw new ZipException( + "ZIP entry has incomplete metadata: " + path + "!" + entry.getName()); + } + } + private static void closeAfterFailure(AutoCloseable closeable, Throwable failure) { try { closeable.close(); @@ -177,4 +299,139 @@ private static void closeAfterFailure(AutoCloseable closeable, Throwable failure failure.addSuppressed(closeException); } } + + static final class ZipEntryReference { + private final String name; + private final int occurrence; + private final boolean directory; + private final int method; + private final long crc; + private final long compressedSize; + private final long expandedSize; + + private ZipEntryReference( + String name, + int occurrence, + boolean directory, + int method, + long crc, + long compressedSize, + long expandedSize) { + this.name = name; + this.occurrence = occurrence; + this.directory = directory; + this.method = method; + this.crc = crc; + this.compressedSize = compressedSize; + this.expandedSize = expandedSize; + } + + String getName() { + return name; + } + + boolean isDirectory() { + return directory; + } + + private boolean matches(ZipArchiveEntry entry) { + return directory == entry.isDirectory() + && method == entry.getMethod() + && crc == entry.getCrc() + && compressedSize == entry.getCompressedSize() + && expandedSize == entry.getSize(); + } + } + + private static final class ValidatedZipInputStream extends InputStream { + private final InputStream inputStream; + private final InputStreamStatistics statistics; + private final ZipFile zipFile; + private final ZipArchiveEntry entry; + private final Path path; + private final CRC32 crc = new CRC32(); + private final byte[] singleByte = new byte[1]; + private long expandedBytes; + private boolean validated; + private boolean closed; + + private ValidatedZipInputStream( + InputStream inputStream, + InputStreamStatistics statistics, + ZipFile zipFile, + ZipArchiveEntry entry, + Path path) { + this.inputStream = inputStream; + this.statistics = statistics; + this.zipFile = zipFile; + this.entry = entry; + this.path = path; + } + + long getCompressedBytesRead() { + return statistics.getCompressedCount(); + } + + @Override + public int read() throws IOException { + int count = read(singleByte, 0, 1); + return count == -1 ? -1 : Byte.toUnsignedInt(singleByte[0]); + } + + @Override + public int read(byte[] bytes, int offset, int length) throws IOException { + int count = inputStream.read(bytes, offset, length); + if (count > 0) { + crc.update(bytes, offset, count); + expandedBytes += count; + } else if (count == -1) { + validate(); + } + return count; + } + + private void validate() throws ZipException { + if (validated) { + return; + } + validated = true; + if (expandedBytes != entry.getSize()) { + throw new ZipException( + "ZIP expanded size does not match central directory: " + path); + } + if (statistics.getCompressedCount() != entry.getCompressedSize()) { + throw new ZipException( + "ZIP compressed size does not match central directory: " + path); + } + if (crc.getValue() != entry.getCrc()) { + throw new ZipException("ZIP CRC does not match central directory: " + path); + } + } + + @Override + public void close() throws IOException { + if (closed) { + return; + } + closed = true; + IOException failure = null; + try { + inputStream.close(); + } catch (IOException exception) { + failure = exception; + } + try { + zipFile.close(); + } catch (IOException exception) { + if (failure == null) { + failure = exception; + } else { + failure.addSuppressed(exception); + } + } + if (failure != null) { + throw failure; + } + } + } } diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/RotatingLogFileMetadata.java b/api/src/main/java/com/microsoft/gctoolkit/io/RotatingLogFileMetadata.java index c203b5a8b..7f3ed328d 100644 --- a/api/src/main/java/com/microsoft/gctoolkit/io/RotatingLogFileMetadata.java +++ b/api/src/main/java/com/microsoft/gctoolkit/io/RotatingLogFileMetadata.java @@ -72,19 +72,18 @@ else if ( isZip()) } private void findZIPSegments(LogFileReadBudget inspectionBudget) { - ZipArchivePreflight.Index index; + List entries; try { - index = ZipArchivePreflight.validate(getPath(), readLimits); + entries = LogFileStreams.zipEntries(getPath(), readLimits); } catch (IOException ioe) { throw new UncheckedIOException(ioe); } - segments = index.getEntries().stream() + segments = entries.stream() .filter(entry -> !entry.isDirectory()) .map(entry -> new GCLogFileZipSegment( getPath(), entry.getName(), readLimits, - index, entry)) .collect(toList()); orderSegments(inspectionBudget); diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/ValidatedZipEntryInputStream.java b/api/src/main/java/com/microsoft/gctoolkit/io/ValidatedZipEntryInputStream.java deleted file mode 100644 index 5adbc93d0..000000000 --- a/api/src/main/java/com/microsoft/gctoolkit/io/ValidatedZipEntryInputStream.java +++ /dev/null @@ -1,277 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. -package com.microsoft.gctoolkit.io; - -import java.io.EOFException; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.nio.file.Path; -import java.util.zip.CRC32; -import java.util.zip.ZipEntry; -import java.util.zip.ZipException; -import java.util.zip.ZipInputStream; - -final class ValidatedZipEntryInputStream extends InputStream { - - private static final long LOCAL_FILE_HEADER = 0x04034b50L; - private static final long DATA_DESCRIPTOR = 0x08074b50L; - private static final int DATA_DESCRIPTOR_FLAG = 8; - - private final InputStream entryInputStream; - private final TrackingZipInputStream trackingZipInputStream; - private final ZipArchivePreflight.Entry expectedEntry; - private final Path path; - private final CRC32 crc = new CRC32(); - private final byte[] singleByte = new byte[1]; - private long expandedBytes; - private boolean validated; - - ValidatedZipEntryInputStream( - InputStream inputStream, - ZipArchivePreflight.Entry expectedEntry, - Path path) throws IOException { - this.expectedEntry = expectedEntry; - this.path = path; - TrackingZipInputStream tracking = null; - try { - if (expectedEntry.getMethod() == ZipEntry.STORED) { - entryInputStream = new StoredEntryInputStream(inputStream, expectedEntry, path); - } else { - tracking = new TrackingZipInputStream(inputStream); - ZipEntry actualEntry = tracking.getNextEntry(); - if (actualEntry == null - || actualEntry.isDirectory() - || actualEntry.getMethod() != expectedEntry.getMethod() - || !expectedEntry.getName().equals(actualEntry.getName())) { - throw new ZipException( - "ZIP local header does not match central directory: " + path); - } - entryInputStream = tracking; - } - trackingZipInputStream = tracking; - } catch (IOException | RuntimeException | Error failure) { - try { - inputStream.close(); - } catch (IOException closeException) { - failure.addSuppressed(closeException); - } - throw failure; - } - } - - long getCompressedBytesRead() { - if (expectedEntry.getMethod() == ZipEntry.STORED) { - return expandedBytes; - } - return trackingZipInputStream.getCompressedBytesRead(); - } - - @Override - public int read() throws IOException { - int count = read(singleByte, 0, 1); - return count == -1 ? -1 : Byte.toUnsignedInt(singleByte[0]); - } - - @Override - public int read(byte[] bytes, int offset, int length) throws IOException { - int count = entryInputStream.read(bytes, offset, length); - if (count > 0) { - crc.update(bytes, offset, count); - expandedBytes += count; - } else if (count == -1) { - validate(); - } - return count; - } - - private void validate() throws ZipException { - if (validated) { - return; - } - validated = true; - if (expandedBytes != expectedEntry.getExpandedSize()) { - throw new ZipException("ZIP expanded size does not match central directory: " + path); - } - if (getCompressedBytesRead() != expectedEntry.getCompressedSize()) { - throw new ZipException("ZIP compressed size does not match central directory: " + path); - } - if (crc.getValue() != expectedEntry.getCrc()) { - throw new ZipException("ZIP CRC does not match central directory: " + path); - } - } - - @Override - public void close() throws IOException { - entryInputStream.close(); - } - - private static final class TrackingZipInputStream extends ZipInputStream { - - private TrackingZipInputStream(InputStream inputStream) { - super(inputStream); - } - - private long getCompressedBytesRead() { - return inf.getBytesRead(); - } - } - - private static final class StoredEntryInputStream extends InputStream { - - private final InputStream inputStream; - private final ZipArchivePreflight.Entry expectedEntry; - private final Path path; - private final boolean hasDescriptor; - private long remaining; - private boolean descriptorValidated; - - private StoredEntryInputStream( - InputStream inputStream, - ZipArchivePreflight.Entry expectedEntry, - Path path) throws IOException { - this.inputStream = inputStream; - this.expectedEntry = expectedEntry; - this.path = path; - if (readUnsignedInt(inputStream, path) != LOCAL_FILE_HEADER) { - throw new ZipException("ZIP local header not found: " + path); - } - ValidatedZipEntryInputStream.skip(inputStream, 2, path); - int flags = readUnsignedShort(inputStream, path); - int method = readUnsignedShort(inputStream, path); - ValidatedZipEntryInputStream.skip(inputStream, 4, path); - long localCrc = readUnsignedInt(inputStream, path); - long localCompressedSize = readUnsignedInt(inputStream, path); - long localExpandedSize = readUnsignedInt(inputStream, path); - int nameLength = readUnsignedShort(inputStream, path); - int extraLength = readUnsignedShort(inputStream, path); - byte[] name = readBytes(inputStream, nameLength, path); - ValidatedZipEntryInputStream.skip(inputStream, extraLength, path); - - if (method != ZipEntry.STORED - || flags != expectedEntry.getFlags() - || !new String(name, StandardCharsets.UTF_8).equals(expectedEntry.getName())) { - throw new ZipException( - "ZIP local header does not match central directory: " + path); - } - hasDescriptor = (flags & DATA_DESCRIPTOR_FLAG) != 0; - if (!hasDescriptor) { - if (localCrc != expectedEntry.getCrc() - || (localCompressedSize != 0xffffffffL - && localCompressedSize != expectedEntry.getCompressedSize()) - || (localExpandedSize != 0xffffffffL - && localExpandedSize != expectedEntry.getExpandedSize())) { - throw new ZipException( - "ZIP local header values do not match central directory: " + path); - } - } - remaining = expectedEntry.getCompressedSize(); - } - - @Override - public int read() throws IOException { - byte[] oneByte = new byte[1]; - int count = read(oneByte, 0, 1); - return count == -1 ? -1 : Byte.toUnsignedInt(oneByte[0]); - } - - @Override - public int read(byte[] bytes, int offset, int length) throws IOException { - if (remaining == 0L) { - validateDescriptor(); - return -1; - } - int count = inputStream.read(bytes, offset, (int) Math.min(length, remaining)); - if (count == -1) { - throw new EOFException("Unexpected end of stored ZIP entry: " + path); - } - remaining -= count; - return count; - } - - private void validateDescriptor() throws IOException { - if (!hasDescriptor || descriptorValidated) { - return; - } - descriptorValidated = true; - long crcOrSignature = readUnsignedInt(inputStream, path); - long descriptorCrc = crcOrSignature == DATA_DESCRIPTOR - ? readUnsignedInt(inputStream, path) - : crcOrSignature; - long compressedSize = expectedEntry.isZip64Size() - ? readLong(inputStream, path) - : readUnsignedInt(inputStream, path); - long expandedSize = expectedEntry.isZip64Size() - ? readLong(inputStream, path) - : readUnsignedInt(inputStream, path); - if (descriptorCrc != expectedEntry.getCrc() - || compressedSize != expectedEntry.getCompressedSize() - || expandedSize != expectedEntry.getExpandedSize()) { - throw new ZipException( - "ZIP data descriptor does not match central directory: " + path); - } - } - - @Override - public void close() throws IOException { - inputStream.close(); - } - } - - private static int readUnsignedShort(InputStream inputStream, Path path) throws IOException { - return readByte(inputStream, path) | (readByte(inputStream, path) << 8); - } - - private static long readUnsignedInt(InputStream inputStream, Path path) throws IOException { - return (long) readByte(inputStream, path) - | ((long) readByte(inputStream, path) << 8) - | ((long) readByte(inputStream, path) << 16) - | ((long) readByte(inputStream, path) << 24); - } - - private static long readLong(InputStream inputStream, Path path) throws IOException { - long value = readUnsignedInt(inputStream, path) - | (readUnsignedInt(inputStream, path) << 32); - if (value < 0L) { - throw new ZipException("ZIP64 value is too large: " + path); - } - return value; - } - - private static int readByte(InputStream inputStream, Path path) throws IOException { - int value = inputStream.read(); - if (value == -1) { - throw new EOFException("Unexpected end of ZIP metadata: " + path); - } - return value; - } - - private static byte[] readBytes( - InputStream inputStream, - int length, - Path path) throws IOException { - byte[] bytes = new byte[length]; - int offset = 0; - while (offset < length) { - int count = inputStream.read(bytes, offset, length - offset); - if (count == -1) { - throw new EOFException("Unexpected end of ZIP metadata: " + path); - } - offset += count; - } - return bytes; - } - - private static void skip(InputStream inputStream, int count, Path path) throws IOException { - int remaining = count; - while (remaining > 0) { - long skipped = inputStream.skip(remaining); - if (skipped == 0L) { - readByte(inputStream, path); - remaining--; - } else { - remaining -= (int) skipped; - } - } - } -} diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/ZipArchivePreflight.java b/api/src/main/java/com/microsoft/gctoolkit/io/ZipArchivePreflight.java index 613fd1616..6d4d08fec 100644 --- a/api/src/main/java/com/microsoft/gctoolkit/io/ZipArchivePreflight.java +++ b/api/src/main/java/com/microsoft/gctoolkit/io/ZipArchivePreflight.java @@ -2,14 +2,12 @@ // Licensed under the MIT License. package com.microsoft.gctoolkit.io; +import java.io.EOFException; import java.io.IOException; -import java.io.RandomAccessFile; -import java.nio.charset.StandardCharsets; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.channels.SeekableByteChannel; import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.zip.ZipEntry; import java.util.zip.ZipException; final class ZipArchivePreflight { @@ -20,75 +18,88 @@ final class ZipArchivePreflight { private static final long CENTRAL_DIRECTORY_ENTRY = 0x02014b50L; private static final long CENTRAL_DIRECTORY_ARCHIVE_EXTRA_DATA = 0x08064b50L; private static final long CENTRAL_DIRECTORY_DIGITAL_SIGNATURE = 0x05054b50L; - private static final int ZIP64_EXTRA_FIELD = 1; private static final int MAXIMUM_COMMENT_LENGTH = 0xffff; private static final int END_OF_CENTRAL_DIRECTORY_SIZE = 22; private ZipArchivePreflight() { } - static Index validate(Path path, LogFileReadLimits limits) throws IOException { - try (RandomAccessFile file = new RandomAccessFile(path.toFile(), "r")) { - if (file.length() > limits.getMaxCompressedBytes()) { - throw new LogFileReadLimitExceededException( - LogFileReadLimitExceededException.LimitType.COMPRESSED_BYTES, + static void validate( + SeekableByteChannel channel, + Path path, + LogFileReadLimits limits) throws IOException { + Reader file = new Reader(channel); + if (file.length() > limits.getMaxCompressedBytes()) { + throw new LogFileReadLimitExceededException( + LogFileReadLimitExceededException.LimitType.COMPRESSED_BYTES, + path, + null, + Long.toString(limits.getMaxCompressedBytes()), + Long.toString(file.length())); + } + + Directory directory = readDirectory(file, path); + if (directory.entryCount > limits.getMaxArchiveEntries()) { + throw limit( + LogFileReadLimitExceededException.LimitType.ARCHIVE_ENTRIES, + path, + limits.getMaxArchiveEntries(), + directory.entryCount); + } + if (directory.size > limits.getMaxArchiveMetadataBytes()) { + throw limit( + LogFileReadLimitExceededException.LimitType.ARCHIVE_METADATA_BYTES, + path, + limits.getMaxArchiveMetadataBytes(), + directory.size); + } + + long directoryEnd = directory.offset + directory.size; + long entryCount = 0L; + long recordCount = 0L; + long maximumRecords = (long) limits.getMaxArchiveEntries() + 2L; + file.seek(directory.offset); + while (file.position() < directoryEnd) { + if (++recordCount > maximumRecords) { + throw limit( + LogFileReadLimitExceededException.LimitType.ARCHIVE_ENTRIES, path, - null, - Long.toString(limits.getMaxCompressedBytes()), - Long.toString(file.length())); - } - Directory directory = readDirectory(file, path); - if (directory.entryCount > limits.getMaxArchiveEntries()) { - throw entryLimit(path, limits.getMaxArchiveEntries(), directory.entryCount); - } - - long directoryEnd = directory.offset + directory.size; - file.seek(directory.offset); - List entries = new ArrayList<>((int) directory.entryCount); - long recordCount = 0L; - long maximumRecords = (long) limits.getMaxArchiveEntries() + 2L; - while (file.getFilePointer() < directoryEnd) { - if (++recordCount > maximumRecords) { - throw entryLimit(path, (int) maximumRecords, recordCount); - } - long signature = readUnsignedInt(file); - if (signature == CENTRAL_DIRECTORY_DIGITAL_SIGNATURE) { - int signatureLength = readUnsignedShort(file); - skip(file, signatureLength, directoryEnd, path); - break; - } - if (signature == CENTRAL_DIRECTORY_ARCHIVE_EXTRA_DATA) { - long extraDataLength = readUnsignedInt(file); - skip(file, extraDataLength, directoryEnd, path); - continue; - } - if (signature != CENTRAL_DIRECTORY_ENTRY) { - throw new ZipException("Invalid ZIP central directory entry: " + path); - } - if (entries.size() == limits.getMaxArchiveEntries()) { - throw entryLimit(path, limits.getMaxArchiveEntries(), entries.size() + 1L); - } - entries.add(readEntry(file, path, directory.offset, directoryEnd)); - } - if (file.getFilePointer() != directoryEnd) { - throw new ZipException("Invalid ZIP central directory size: " + path); + maximumRecords, + recordCount); } - if (entries.size() != directory.entryCount) { - throw new ZipException("ZIP central directory entry count mismatch: " + path); + long signature = file.readUnsignedInt(); + if (signature == CENTRAL_DIRECTORY_ENTRY) { + entryCount++; + file.skip(24L, directoryEnd, path); + int nameLength = file.readUnsignedShort(); + int extraLength = file.readUnsignedShort(); + int commentLength = file.readUnsignedShort(); + file.skip( + 12L + nameLength + extraLength + commentLength, + directoryEnd, + path); + } else if (signature == CENTRAL_DIRECTORY_ARCHIVE_EXTRA_DATA) { + file.skip(file.readUnsignedInt(), directoryEnd, path); + } else if (signature == CENTRAL_DIRECTORY_DIGITAL_SIGNATURE) { + file.skip(file.readUnsignedShort(), directoryEnd, path); + } else { + throw new ZipException("Invalid ZIP central directory record: " + path); } - return new Index(directory.offset, entries); + } + if (file.position() != directoryEnd || entryCount != directory.entryCount) { + throw new ZipException("ZIP central directory does not match its declared size: " + path); } } - private static Directory readDirectory(RandomAccessFile file, Path path) throws IOException { + private static Directory readDirectory(Reader file, Path path) throws IOException { long endOffset = findEndOfCentralDirectory(file); - file.seek(endOffset + 4); - int diskNumber = readUnsignedShort(file); - int centralDirectoryDisk = readUnsignedShort(file); - long entriesOnDisk = readUnsignedShort(file); - long totalEntries = readUnsignedShort(file); - long centralDirectorySize = readUnsignedInt(file); - long centralDirectoryOffset = readUnsignedInt(file); + file.seek(endOffset + 4L); + int diskNumber = file.readUnsignedShort(); + int centralDirectoryDisk = file.readUnsignedShort(); + long entriesOnDisk = file.readUnsignedShort(); + long totalEntries = file.readUnsignedShort(); + long centralDirectorySize = file.readUnsignedInt(); + long centralDirectoryOffset = file.readUnsignedInt(); if (diskNumber != 0 || centralDirectoryDisk != 0 || entriesOnDisk != totalEntries) { throw new ZipException("Split ZIP archives are not supported: " + path); @@ -106,172 +117,17 @@ private static Directory readDirectory(RandomAccessFile file, Path path) throws centralDirectoryOffset); } - private static Entry readEntry( - RandomAccessFile file, - Path path, - long centralDirectoryOffset, - long directoryEnd) throws IOException { - skip(file, 4, directoryEnd, path); - int flags = readUnsignedShort(file); - int method = readUnsignedShort(file); - skip(file, 4, directoryEnd, path); - long crc = readUnsignedInt(file); - long compressedSize = readUnsignedInt(file); - long expandedSize = readUnsignedInt(file); - boolean zip64Size = compressedSize == 0xffffffffL || expandedSize == 0xffffffffL; - int nameLength = readUnsignedShort(file); - int extraLength = readUnsignedShort(file); - int commentLength = readUnsignedShort(file); - int diskStart = readUnsignedShort(file); - skip(file, 6, directoryEnd, path); - long localHeaderOffset = readUnsignedInt(file); - - byte[] nameBytes = new byte[nameLength]; - file.readFully(nameBytes); - byte[] extra = new byte[extraLength]; - file.readFully(extra); - skip(file, commentLength, directoryEnd, path); - - long[] zip64 = parseZip64Extra( - extra, - expandedSize == 0xffffffffL, - compressedSize == 0xffffffffL, - localHeaderOffset == 0xffffffffL, - diskStart == 0xffff, - path); - int zip64Index = 0; - if (expandedSize == 0xffffffffL) { - expandedSize = zip64[zip64Index++]; - } - if (compressedSize == 0xffffffffL) { - compressedSize = zip64[zip64Index++]; - } - if (localHeaderOffset == 0xffffffffL) { - localHeaderOffset = zip64[zip64Index++]; - } - if (diskStart == 0xffff) { - diskStart = Math.toIntExact(zip64[zip64Index]); - } - if (diskStart != 0) { - throw new ZipException("Split ZIP entries are not supported: " + path); - } - if (localHeaderOffset < 0L || localHeaderOffset >= centralDirectoryOffset) { - throw new ZipException("Invalid ZIP local header offset: " + path); - } - if ((flags & 1) != 0) { - throw new ZipException("Encrypted ZIP entries are not supported: " + path); - } - if (method != ZipEntry.STORED && method != ZipEntry.DEFLATED) { - throw new ZipException("Unsupported ZIP compression method: " + method); - } - - String name = new String(nameBytes, StandardCharsets.UTF_8); - return new Entry( - name, - name.endsWith("/"), - flags, - method, - crc, - compressedSize, - expandedSize, - localHeaderOffset, - zip64Size); - } - - private static long[] parseZip64Extra( - byte[] extra, - boolean expandedSizeRequired, - boolean compressedSizeRequired, - boolean offsetRequired, - boolean diskRequired, - Path path) throws ZipException { - int requiredValues = (expandedSizeRequired ? 1 : 0) - + (compressedSizeRequired ? 1 : 0) - + (offsetRequired ? 1 : 0) - + (diskRequired ? 1 : 0); - if (requiredValues == 0) { - return new long[0]; - } - - int offset = 0; - while (offset <= extra.length - 4) { - int id = unsignedShort(extra, offset); - int size = unsignedShort(extra, offset + 2); - offset += 4; - if (size > extra.length - offset) { - throw new ZipException("Invalid ZIP extra field: " + path); - } - if (id == ZIP64_EXTRA_FIELD) { - List values = new ArrayList<>(requiredValues); - int valueOffset = offset; - int fieldEnd = offset + size; - if (expandedSizeRequired) { - values.add(littleEndianLong(extra, valueOffset, fieldEnd, path)); - valueOffset += 8; - } - if (compressedSizeRequired) { - values.add(littleEndianLong(extra, valueOffset, fieldEnd, path)); - valueOffset += 8; - } - if (offsetRequired) { - values.add(littleEndianLong(extra, valueOffset, fieldEnd, path)); - valueOffset += 8; - } - if (diskRequired) { - if (valueOffset > fieldEnd - 4) { - throw new ZipException("Invalid ZIP64 extra field: " + path); - } - values.add(unsignedInt(extra, valueOffset)); - } - long[] result = new long[values.size()]; - for (int index = 0; index < result.length; index++) { - result[index] = values.get(index); - } - return result; - } - offset += size; - } - throw new ZipException("Required ZIP64 extra field not found: " + path); - } - - private static long littleEndianLong( - byte[] bytes, - int offset, - int fieldEnd, - Path path) throws ZipException { - if (offset > fieldEnd - 8 || offset > bytes.length - 8) { - throw new ZipException("Invalid ZIP64 extra field: " + path); - } - long low = unsignedInt(bytes, offset); - long high = unsignedInt(bytes, offset + 4); - long value = low | (high << 32); - if (value < 0L) { - throw new ZipException("ZIP64 value is too large: " + path); - } - return value; - } - - private static int unsignedShort(byte[] bytes, int offset) { - return Byte.toUnsignedInt(bytes[offset]) | (Byte.toUnsignedInt(bytes[offset + 1]) << 8); - } - - private static long unsignedInt(byte[] bytes, int offset) { - return (long) Byte.toUnsignedInt(bytes[offset]) - | ((long) Byte.toUnsignedInt(bytes[offset + 1]) << 8) - | ((long) Byte.toUnsignedInt(bytes[offset + 2]) << 16) - | ((long) Byte.toUnsignedInt(bytes[offset + 3]) << 24); - } - - private static long findEndOfCentralDirectory(RandomAccessFile file) throws IOException { - long minimumOffset = Math.max(0L, + private static long findEndOfCentralDirectory(Reader file) throws IOException { + long minimumOffset = Math.max( + 0L, file.length() - END_OF_CENTRAL_DIRECTORY_SIZE - MAXIMUM_COMMENT_LENGTH); for (long offset = file.length() - END_OF_CENTRAL_DIRECTORY_SIZE; offset >= minimumOffset; offset--) { file.seek(offset); - if (readUnsignedInt(file) == END_OF_CENTRAL_DIRECTORY) { - file.seek(offset + 20); - int commentLength = readUnsignedShort(file); + if (file.readUnsignedInt() == END_OF_CENTRAL_DIRECTORY) { + file.seek(offset + 20L); + int commentLength = file.readUnsignedShort(); if (offset + END_OF_CENTRAL_DIRECTORY_SIZE + commentLength == file.length()) { return offset; } @@ -281,7 +137,7 @@ private static long findEndOfCentralDirectory(RandomAccessFile file) throws IOEx } private static Directory readZip64Directory( - RandomAccessFile file, + Reader file, Path path, long endOffset) throws IOException { long locatorOffset = endOffset - 20L; @@ -289,32 +145,35 @@ private static Directory readZip64Directory( throw new ZipException("ZIP64 locator not found: " + path); } file.seek(locatorOffset); - if (readUnsignedInt(file) != ZIP64_END_OF_CENTRAL_DIRECTORY_LOCATOR) { - throw new ZipException("ZIP64 locator not found: " + path); - } - if (readUnsignedInt(file) != 0L) { - throw new ZipException("Split ZIP64 archives are not supported: " + path); - } - long zip64Offset = readLong(file); - if (readUnsignedInt(file) != 1L || zip64Offset < 0L || zip64Offset > file.length() - 56L) { + if (file.readUnsignedInt() != ZIP64_END_OF_CENTRAL_DIRECTORY_LOCATOR + || file.readUnsignedInt() != 0L) { + throw new ZipException("Invalid or split ZIP64 locator: " + path); + } + long zip64Offset = file.readLong(); + if (file.readUnsignedInt() != 1L + || zip64Offset < 0L + || zip64Offset > file.length() - 56L) { throw new ZipException("Invalid ZIP64 locator: " + path); } + file.seek(zip64Offset); - if (readUnsignedInt(file) != ZIP64_END_OF_CENTRAL_DIRECTORY) { + if (file.readUnsignedInt() != ZIP64_END_OF_CENTRAL_DIRECTORY) { throw new ZipException("ZIP64 end of central directory not found: " + path); } - long recordSize = readLong(file); + long recordSize = file.readLong(); if (recordSize < 44L) { throw new ZipException("Invalid ZIP64 end of central directory: " + path); } - skip(file, 4, file.length(), path); - long diskNumber = readUnsignedInt(file); - long centralDirectoryDisk = readUnsignedInt(file); - long entriesOnDisk = readLong(file); - long totalEntries = readLong(file); - long centralDirectorySize = readLong(file); - long centralDirectoryOffset = readLong(file); - if (diskNumber != 0L || centralDirectoryDisk != 0L || entriesOnDisk != totalEntries) { + file.skip(4L, file.length(), path); + long diskNumber = file.readUnsignedInt(); + long centralDirectoryDisk = file.readUnsignedInt(); + long entriesOnDisk = file.readLong(); + long totalEntries = file.readLong(); + long centralDirectorySize = file.readLong(); + long centralDirectoryOffset = file.readLong(); + if (diskNumber != 0L + || centralDirectoryDisk != 0L + || entriesOnDisk != totalEntries) { throw new ZipException("Split ZIP64 archives are not supported: " + path); } return checkedDirectory( @@ -337,142 +196,77 @@ private static Directory checkedDirectory( return new Directory(entryCount, size, offset); } - private static int readUnsignedShort(RandomAccessFile file) throws IOException { - return file.readUnsignedByte() | (file.readUnsignedByte() << 8); - } - - private static long readUnsignedInt(RandomAccessFile file) throws IOException { - return (long) file.readUnsignedByte() - | ((long) file.readUnsignedByte() << 8) - | ((long) file.readUnsignedByte() << 16) - | ((long) file.readUnsignedByte() << 24); - } - - private static long readLong(RandomAccessFile file) throws IOException { - long value = readUnsignedInt(file) | (readUnsignedInt(file) << 32); - if (value < 0L) { - throw new ZipException("ZIP64 value is too large"); - } - return value; - } - - private static void skip( - RandomAccessFile file, - long count, - long maximumOffset, - Path path) throws IOException { - long current = file.getFilePointer(); - long target = current + count; - if (count < 0L || target < current || target > maximumOffset) { - throw new ZipException("Invalid ZIP central directory field length: " + path); - } - file.seek(target); - } - - private static LogFileReadLimitExceededException entryLimit( + private static LogFileReadLimitExceededException limit( + LogFileReadLimitExceededException.LimitType limitType, Path path, - int maximumEntries, - long observedEntries) { + long configured, + long observed) { return new LogFileReadLimitExceededException( - LogFileReadLimitExceededException.LimitType.ARCHIVE_ENTRIES, + limitType, path, null, - Integer.toString(maximumEntries), - Long.toString(observedEntries)); + Long.toString(configured), + Long.toString(observed)); } - static final class Index { - private final long centralDirectoryOffset; - private final List entries; - - private Index(long centralDirectoryOffset, List entries) { - this.centralDirectoryOffset = centralDirectoryOffset; - this.entries = Collections.unmodifiableList(entries); - } - - long getCentralDirectoryOffset() { - return centralDirectoryOffset; - } - - List getEntries() { - return entries; - } + private static final class Reader { + private final SeekableByteChannel channel; + private final ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES) + .order(ByteOrder.LITTLE_ENDIAN); - Entry firstFile() { - return entries.stream().filter(entry -> !entry.isDirectory()).findFirst().orElse(null); + private Reader(SeekableByteChannel channel) { + this.channel = channel; } - Entry find(String name) { - return entries.stream().filter(entry -> entry.getName().equals(name)).findFirst().orElse(null); + private long length() throws IOException { + return channel.size(); } - } - static final class Entry { - private final String name; - private final boolean directory; - private final int flags; - private final int method; - private final long crc; - private final long compressedSize; - private final long expandedSize; - private final long localHeaderOffset; - private final boolean zip64Size; - - private Entry( - String name, - boolean directory, - int flags, - int method, - long crc, - long compressedSize, - long expandedSize, - long localHeaderOffset, - boolean zip64Size) { - this.name = name; - this.directory = directory; - this.flags = flags; - this.method = method; - this.crc = crc; - this.compressedSize = compressedSize; - this.expandedSize = expandedSize; - this.localHeaderOffset = localHeaderOffset; - this.zip64Size = zip64Size; + private long position() throws IOException { + return channel.position(); } - String getName() { - return name; + private void seek(long position) throws IOException { + channel.position(position); } - boolean isDirectory() { - return directory; + private int readUnsignedShort() throws IOException { + read(Short.BYTES); + return Short.toUnsignedInt(buffer.getShort()); } - int getMethod() { - return method; + private long readUnsignedInt() throws IOException { + read(Integer.BYTES); + return Integer.toUnsignedLong(buffer.getInt()); } - int getFlags() { - return flags; - } - - long getCrc() { - return crc; - } - - long getCompressedSize() { - return compressedSize; - } - - long getExpandedSize() { - return expandedSize; + private long readLong() throws IOException { + read(Long.BYTES); + long value = buffer.getLong(); + if (value < 0L) { + throw new ZipException("ZIP64 value is too large"); + } + return value; } - long getLocalHeaderOffset() { - return localHeaderOffset; + private void read(int length) throws IOException { + buffer.clear(); + buffer.limit(length); + while (buffer.hasRemaining()) { + if (channel.read(buffer) == -1) { + throw new EOFException("Unexpected end of ZIP metadata"); + } + } + buffer.flip(); } - boolean isZip64Size() { - return zip64Size; + private void skip(long count, long maximumOffset, Path path) throws IOException { + long current = position(); + long target = current + count; + if (count < 0L || target < current || target > maximumOffset) { + throw new ZipException("Invalid ZIP central directory field length: " + path); + } + seek(target); } } diff --git a/api/src/main/java/module-info.java b/api/src/main/java/module-info.java index 8afa6d468..31437fd43 100644 --- a/api/src/main/java/module-info.java +++ b/api/src/main/java/module-info.java @@ -31,6 +31,7 @@ */ module com.microsoft.gctoolkit.api { requires java.logging; + requires org.apache.commons.compress; exports com.microsoft.gctoolkit; exports com.microsoft.gctoolkit.aggregator; diff --git a/api/src/test/java/com/microsoft/gctoolkit/io/SingleGCLogFileReadLimitsTest.java b/api/src/test/java/com/microsoft/gctoolkit/io/SingleGCLogFileReadLimitsTest.java index 165130873..5e84d1a1d 100644 --- a/api/src/test/java/com/microsoft/gctoolkit/io/SingleGCLogFileReadLimitsTest.java +++ b/api/src/test/java/com/microsoft/gctoolkit/io/SingleGCLogFileReadLimitsTest.java @@ -31,6 +31,7 @@ import static com.microsoft.gctoolkit.io.GCLogFile.END_OF_DATA_SENTINEL; import static com.microsoft.gctoolkit.io.LogFileReadLimitExceededException.LimitType.ARCHIVE_ENTRIES; +import static com.microsoft.gctoolkit.io.LogFileReadLimitExceededException.LimitType.ARCHIVE_METADATA_BYTES; import static com.microsoft.gctoolkit.io.LogFileReadLimitExceededException.LimitType.COMPRESSED_BYTES; import static com.microsoft.gctoolkit.io.LogFileReadLimitExceededException.LimitType.COMPRESSED_MEMBERS; import static com.microsoft.gctoolkit.io.LogFileReadLimitExceededException.LimitType.COMPRESSION_RATIO; @@ -413,11 +414,7 @@ void rejectsZipWithForgedCentralDirectoryCompressedSize() throws IOException { Files.write(log, archive); LogFileReadLimits limits = new LogFileReadLimits(16 * 1024, 16 * 1024, 10_000.0d, 1024, 16); - assertThrows(UncheckedIOException.class, () -> { - try (Stream stream = new SingleGCLogFile(log, limits).stream()) { - stream.count(); - } - }); + assertThrows(IOException.class, () -> new SingleGCLogFile(log, limits).stream()); } @Test @@ -452,6 +449,23 @@ void rejectsExcessiveZipControlRecords() throws IOException { assertLimit(ARCHIVE_ENTRIES, () -> new SingleGCLogFile(log, limits).stream()); } + @Test + void rejectsOversizedZipCentralDirectoryBeforeIndexing() throws IOException { + Path log = temporaryDirectory.resolve("central-directory.zip"); + writeZip(log, Map.of("gc.log", "line\n")); + LogFileReadLimits limits = new LogFileReadLimits( + 64 * 1024, + 64 * 1024, + 1024, + 100.0d, + 1024, + 16, + 32, + 1024); + + assertLimit(ARCHIVE_METADATA_BYTES, () -> new SingleGCLogFile(log, limits).stream()); + } + @Test void rejectsExcessiveDirectorySegments() throws IOException { Path directory = temporaryDirectory.resolve("rotating"); @@ -551,6 +565,8 @@ void validatesLimitConfiguration() { () -> new LogFileReadLimits(0, 1, 1, 1.0d, 1, 1, 1)); assertThrows(IllegalArgumentException.class, () -> new LogFileReadLimits(1, 1, 1, 1.0d, 1, 1, 0)); + assertThrows(IllegalArgumentException.class, + () -> new LogFileReadLimits(1, 1, 1, 1.0d, 1, 1, 0, 1)); } @Test @@ -563,6 +579,7 @@ void exposesFiniteSecureDefaults() { assertTrue(Double.isFinite(defaults.getMaxCompressionRatio())); assertTrue(defaults.getCompressionRatioGraceBytes() > 0); assertTrue(defaults.getMaxArchiveEntries() > 0); + assertTrue(defaults.getMaxArchiveMetadataBytes() > 0); assertTrue(defaults.getMaxGzipHeaderBytes() > 0); } diff --git a/pom.xml b/pom.xml index 1e69d2be5..3fea3f968 100644 --- a/pom.xml +++ b/pom.xml @@ -46,6 +46,7 @@ 3.0.3-SNAPSHOT 13.10.0 + 1.28.0 1.25.0 5.13.4 3.2.0 @@ -124,6 +125,11 @@ gctoolkit-gclogs ${project.version} + + org.apache.commons + commons-compress + ${commons-compress.version} + org.junit.jupiter junit-jupiter-engine From f06815e4679a6af9ceab86a385bf79007e498ee9 Mon Sep 17 00:00:00 2001 From: Derek Keeler Date: Wed, 2 Sep 2026 14:56:03 -0700 Subject: [PATCH 3/3] fix(api): accommodate valid large GC logs Raise the finite expanded-byte and line-length defaults above the largest existing regression fixtures while retaining independent compression and archive safeguards. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../java/com/microsoft/gctoolkit/io/LogFileReadLimits.java | 4 ++-- .../microsoft/gctoolkit/io/SingleGCLogFileReadLimitsTest.java | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/LogFileReadLimits.java b/api/src/main/java/com/microsoft/gctoolkit/io/LogFileReadLimits.java index 87122af53..b83ecd235 100644 --- a/api/src/main/java/com/microsoft/gctoolkit/io/LogFileReadLimits.java +++ b/api/src/main/java/com/microsoft/gctoolkit/io/LogFileReadLimits.java @@ -13,13 +13,13 @@ public final class LogFileReadLimits { /** Default maximum number of bytes read from plaintext or emitted by decompression. */ - public static final long DEFAULT_MAX_EXPANDED_BYTES = 1024L * 1024L * 1024L; + public static final long DEFAULT_MAX_EXPANDED_BYTES = 2L * 1024L * 1024L * 1024L; /** Default maximum number of bytes read from one compressed input. */ public static final long DEFAULT_MAX_COMPRESSED_BYTES = 1024L * 1024L * 1024L; /** Default maximum number of decoded characters in one line. */ - public static final int DEFAULT_MAX_LINE_CHARACTERS = 1024 * 1024; + public static final int DEFAULT_MAX_LINE_CHARACTERS = 2 * 1024 * 1024; /** Default maximum ratio of expanded bytes to compressed bytes. */ public static final double DEFAULT_MAX_COMPRESSION_RATIO = 100.0d; diff --git a/api/src/test/java/com/microsoft/gctoolkit/io/SingleGCLogFileReadLimitsTest.java b/api/src/test/java/com/microsoft/gctoolkit/io/SingleGCLogFileReadLimitsTest.java index 5e84d1a1d..4bc00af25 100644 --- a/api/src/test/java/com/microsoft/gctoolkit/io/SingleGCLogFileReadLimitsTest.java +++ b/api/src/test/java/com/microsoft/gctoolkit/io/SingleGCLogFileReadLimitsTest.java @@ -46,7 +46,8 @@ class SingleGCLogFileReadLimitsTest { - private static final int DEFAULT_MAX_LINE_CHARACTERS = 1024 * 1024; + private static final int DEFAULT_MAX_LINE_CHARACTERS = + LogFileReadLimits.DEFAULT_MAX_LINE_CHARACTERS; private static final int HIGHLY_COMPRESSIBLE_LINE_COUNT = 2048; private static final String HIGHLY_COMPRESSIBLE_LINE = "a".repeat(1024); private static final LogFileReadLimits SMALL_LIMITS =