diff --git a/java/common/src/main/resources/org/apache/tsfile/i18n/messages.properties b/java/common/src/main/resources/org/apache/tsfile/i18n/messages.properties index 412aea9d2..3700a46f6 100644 --- a/java/common/src/main/resources/org/apache/tsfile/i18n/messages.properties +++ b/java/common/src/main/resources/org/apache/tsfile/i18n/messages.properties @@ -1623,3 +1623,4 @@ error.encrypt.chunk_authentication_unsupported = Provider %1$s does not support error.encrypt.invalid_chunk_authentication = The encrypted Chunk position authentication proof is missing or has an invalid size error.encrypt.invalid_chunk_authentication_size = Invalid chunk authentication size: %1$d error.encrypt.chunk_authentication_failed = Chunk position authentication failed +error.encrypt.invalid_page_body_lengths = Invalid authenticated page body length encoding diff --git a/java/common/src/main/resources/org/apache/tsfile/i18n/messages_zh.properties b/java/common/src/main/resources/org/apache/tsfile/i18n/messages_zh.properties index 6df14426b..e86263ab0 100644 --- a/java/common/src/main/resources/org/apache/tsfile/i18n/messages_zh.properties +++ b/java/common/src/main/resources/org/apache/tsfile/i18n/messages_zh.properties @@ -1623,3 +1623,4 @@ error.encrypt.chunk_authentication_unsupported = Provider %1$s 不支持 Chunk error.encrypt.invalid_chunk_authentication = 加密 Chunk 的位置认证凭据缺失或长度无效 error.encrypt.invalid_chunk_authentication_size = 无效的 Chunk 认证长度:%1$d error.encrypt.chunk_authentication_failed = Chunk 位置认证失败 +error.encrypt.invalid_page_body_lengths = 认证 Page 正文长度编码无效 diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/EncryptParameter.java b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/EncryptParameter.java index 43215ab7a..815bec553 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/EncryptParameter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/EncryptParameter.java @@ -235,6 +235,24 @@ public boolean isTdePageAead() { return pageAead; } + public int getEncryptedPageBodySize(int plaintextSize) { + int size = getOrCreateFileEncrypt().getEncryptedPageBodySize(plaintextSize); + if (plaintextSize < 0 || size <= plaintextSize) { + throw new EncryptException(Messages.get("error.encrypt.invalid_page_body_lengths")); + } + return size; + } + + public int getPagePlaintextSize(ByteBuffer encryptedBody) { + int size = getOrCreateFileEncrypt().getPagePlaintextSize(encryptedBody.asReadOnlyBuffer()); + if (size < 0 + || size >= encryptedBody.remaining() + || getEncryptedPageBodySize(size) != encryptedBody.remaining()) { + throw new EncryptException(Messages.get("error.encrypt.invalid_page_body_lengths")); + } + return size; + } + public int getChunkAuthenticationSize() { return getOrCreateFileEncrypt().getChunkAuthenticationSize(); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/IEncrypt.java b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/IEncrypt.java index 4651b231c..66d33d014 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/IEncrypt.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/IEncrypt.java @@ -21,6 +21,7 @@ import org.apache.tsfile.exception.encrypt.EncryptException; import org.apache.tsfile.i18n.Messages; +import java.nio.ByteBuffer; import java.util.concurrent.ConcurrentHashMap; public interface IEncrypt extends AutoCloseable { @@ -44,6 +45,28 @@ default int getPageBodyOverhead() { return -1; } + /** Returns the exact encoded size, including profile-specific framing and padding. */ + default int getEncryptedPageBodySize(int plaintextSize) { + int overhead = getPageBodyOverhead(); + if (plaintextSize < 0 || overhead <= 0) { + throw new EncryptException(Messages.get("error.encrypt.invalid_page_body_lengths")); + } + return Math.addExact(plaintextSize, overhead); + } + + /** + * Returns the candidate compressed plaintext size from a bounded, read-only encoded body. This is + * untrusted framing until decryptPage authenticates it together with the page context. + * Implementations must not decrypt, allocate based on this value, or modify the input. + */ + default int getPagePlaintextSize(ByteBuffer encryptedBody) { + int overhead = getPageBodyOverhead(); + if (overhead <= 0 || encryptedBody.remaining() < overhead) { + throw new EncryptException(Messages.get("error.encrypt.invalid_page_body_lengths")); + } + return encryptedBody.remaining() - overhead; + } + /** * Returns this immutable profile's chunk-position proof size. Page-AEAD providers must override * this method; unsupported providers fail closed. The size must be positive and no larger than diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/PageAeadEncryption.java b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/PageAeadEncryption.java index ba53dedcf..c0d193432 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/PageAeadEncryption.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/PageAeadEncryption.java @@ -22,6 +22,7 @@ import org.apache.tsfile.file.metadata.enums.EncryptionType; import org.apache.tsfile.i18n.Messages; +import java.nio.ByteBuffer; import java.util.concurrent.atomic.AtomicBoolean; /** Enforces page-only, fail-closed semantics independently of a provider's legacy methods. */ @@ -121,6 +122,18 @@ public int getPageBodyOverhead() { return delegate.getPageBodyOverhead(); } + @Override + public int getEncryptedPageBodySize(int plaintextSize) { + ensureOpen(); + return delegate.getEncryptedPageBodySize(plaintextSize); + } + + @Override + public int getPagePlaintextSize(ByteBuffer encryptedBody) { + ensureOpen(); + return delegate.getPagePlaintextSize(encryptedBody.asReadOnlyBuffer()); + } + @Override public int getChunkAuthenticationSize() { ensureOpen(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/PageCryptoContext.java b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/PageCryptoContext.java index 6785be999..3b3628cf3 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/PageCryptoContext.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/PageCryptoContext.java @@ -67,8 +67,7 @@ public static PageCryptoContext forEncryption( int pageIndex, long chunkOrdinal) { validate(parameter, uncompressedSize, compressedPlaintextSize, pageIndex, chunkOrdinal); - int encryptedPageBodySize = - Math.addExact(compressedPlaintextSize, parameter.getPageBodyOverhead()); + int encryptedPageBodySize = parameter.getEncryptedPageBodySize(compressedPlaintextSize); return new PageCryptoContext( parameter, uncompressedSize, @@ -104,6 +103,24 @@ public int getUncompressedSize() { return uncompressedSize; } + /** Builds AAD using profile framing; the candidate length is authenticated by decryptPage. */ + public static PageCryptoContext forDecryption( + EncryptParameter parameter, + int uncompressedSize, + ByteBuffer encryptedBody, + int pageIndex, + long chunkOrdinal) { + validate(parameter, uncompressedSize, 0, pageIndex, chunkOrdinal); + int plaintextSize = parameter.getPagePlaintextSize(encryptedBody); + return new PageCryptoContext( + parameter, + uncompressedSize, + plaintextSize, + encryptedBody.remaining(), + pageIndex, + chunkOrdinal); + } + /** Returns the compressed plaintext size on both encryption and decryption paths. */ public int getInputSize() { return compressedPlaintextSize; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java index 24ef984c6..a32f78321 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java @@ -2402,7 +2402,7 @@ private static ByteBuffer decrypt( PageCryptoContext.forDecryption( encryptParameter, header.getUncompressedSize(), - buffer.remaining(), + buffer.asReadOnlyBuffer(), pageIndex, chunkOrdinal); byte[] plaintext = diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/ChunkReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/ChunkReader.java index 64bae1f0a..ff4c68d7c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/ChunkReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/ChunkReader.java @@ -273,7 +273,7 @@ public static ByteBuffer decryptAndUncompressPageData( PageCryptoContext.forDecryption( encryptParameter, pageHeader.getUncompressedSize(), - compressedPageBodyLength, + compressedPageData.slice().limit(compressedPageBodyLength), pageIndex, chunkOrdinal); byte[] decryptedPageData = diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/LazyLoadPageData.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/LazyLoadPageData.java index acde41003..3eea62b90 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/LazyLoadPageData.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/LazyLoadPageData.java @@ -89,7 +89,7 @@ public ByteBuffer uncompressPageData(PageHeader pageHeader) throws IOException { PageCryptoContext.forDecryption( encryptParam, pageHeader.getUncompressedSize(), - compressedPageBodyLength, + ByteBuffer.wrap(chunkData, pageDataOffset, compressedPageBodyLength).slice(), pageIndex, chunkOrdinal); decryptedPageData = diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encrypt/PageCryptoContextTest.java b/java/tsfile/src/test/java/org/apache/tsfile/encrypt/PageCryptoContextTest.java index 947e681b1..efd4f3a46 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/encrypt/PageCryptoContextTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/encrypt/PageCryptoContextTest.java @@ -29,6 +29,7 @@ import java.io.ByteArrayOutputStream; import java.io.DataOutputStream; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -106,6 +107,44 @@ public void testAeadRejectsTamperingAndPageSwap() { } } + @Test + public void testPaddedLengthFramingAndAuthenticatedCandidate() { + try (EncryptParameter parameter = + TestAeadEncryptionProvider.createPaddedParameter(new byte[16], new byte[16])) { + for (int size : new int[] {0, 1, 15, 16, 17, 31, 32, 65536}) { + byte[] plaintext = new byte[size]; + PageCryptoContext write = PageCryptoContext.forEncryption(parameter, size, size, 0, 2); + byte[] body = IEncryptor.getEncryptor(parameter).encryptPage(plaintext, 0, size, write); + assertEquals(write.getEncryptedPageBodySize(), body.length); + ByteBuffer slice = ByteBuffer.allocateDirect(body.length + 9); + slice.position(5); + slice.put(body).flip().position(5); + PageCryptoContext read = PageCryptoContext.forDecryption(parameter, size, slice, 0, 2); + assertEquals(5, slice.position()); + assertEquals(size, read.getCompressedPlaintextSize()); + assertArrayEquals(write.getAssociatedData(), read.getAssociatedData()); + assertArrayEquals( + plaintext, IDecryptor.getDecryptor(parameter).decryptPage(body, 0, body.length, read)); + if (size > 1 && size % 16 != 0) { + ByteBuffer.wrap(body).putInt(size - 1); + PageCryptoContext modified = + PageCryptoContext.forDecryption(parameter, size, ByteBuffer.wrap(body), 0, 2); + assertThrows( + EncryptException.class, + () -> IDecryptor.getDecryptor(parameter).decryptPage(body, 0, body.length, modified)); + } + } + assertThrows( + EncryptException.class, + () -> PageCryptoContext.forDecryption(parameter, 1, ByteBuffer.allocate(3), 0, 2)); + assertThrows( + EncryptException.class, + () -> + PageCryptoContext.forDecryption( + parameter, 1, ByteBuffer.allocate(48).putInt(-1).flip(), 0, 2)); + } + } + @Test public void testChunkOrdinalAllocationSurvivesCopyAndRecovery() { EncryptParameter parameter = diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encrypt/TestAeadEncryptionProvider.java b/java/tsfile/src/test/java/org/apache/tsfile/encrypt/TestAeadEncryptionProvider.java index 752928fba..9ec770392 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/encrypt/TestAeadEncryptionProvider.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/encrypt/TestAeadEncryptionProvider.java @@ -39,6 +39,7 @@ public final class TestAeadEncryptionProvider implements IEncryptProvider { public static final String PROVIDER_ID = "test-aead-provider"; public static final String PROFILE_ID = "AES_GCM_128_AES_WRAP_128_V1"; public static final String MAC_PROFILE_ID = "TEST_AES_GCM_HMAC_SHA256_V1"; + public static final String PADDED_PROFILE_ID = "TEST_PADDED_AES_GCM_V1"; private static final int IV_LENGTH = 12; private static final int TAG_LENGTH = 16; @@ -62,12 +63,15 @@ public String getProviderId() { @Override public IEncrypt create(EncryptParameter encryptParameter) { if (!PROFILE_ID.equals(encryptParameter.getProfileId()) - && !MAC_PROFILE_ID.equals(encryptParameter.getProfileId())) { + && !MAC_PROFILE_ID.equals(encryptParameter.getProfileId()) + && !PADDED_PROFILE_ID.equals(encryptParameter.getProfileId())) { throw new EncryptException("unsupported test encryption profile"); } CREATE_COUNT.incrementAndGet(); return new TestAeadEncrypt( - resolveDataKey(encryptParameter), MAC_PROFILE_ID.equals(encryptParameter.getProfileId())); + resolveDataKey(encryptParameter), + MAC_PROFILE_ID.equals(encryptParameter.getProfileId()), + PADDED_PROFILE_ID.equals(encryptParameter.getProfileId())); } public static EncryptParameter createParameter(byte[] dataKey, byte[] fileCryptoId) { @@ -79,6 +83,10 @@ public static EncryptParameter createMacParameter(byte[] dataKey, byte[] fileCry return createParameter(dataKey, fileCryptoId, MAC_PROFILE_ID); } + public static EncryptParameter createPaddedParameter(byte[] dataKey, byte[] fileCryptoId) { + return createParameter(dataKey, fileCryptoId, PADDED_PROFILE_ID); + } + private static EncryptParameter createParameter( byte[] dataKey, byte[] fileCryptoId, String profile) { return EncryptParameter.pageAeadBuilder() @@ -134,25 +142,45 @@ private static final class TestAeadEncrypt implements IEncrypt { private final byte[] dataKey; private final boolean macProof; + private final boolean padded; - private TestAeadEncrypt(byte[] dataKey, boolean macProof) { + private TestAeadEncrypt(byte[] dataKey, boolean macProof, boolean padded) { this.dataKey = dataKey; this.macProof = macProof; + this.padded = padded; } @Override public IDecryptor getDecryptor() { - return new AeadDecryptor(dataKey); + return new AeadDecryptor(dataKey, padded); } @Override public IEncryptor getEncryptor() { - return new AeadEncryptor(dataKey); + return new AeadEncryptor(dataKey, padded); } @Override public int getPageBodyOverhead() { - return IV_LENGTH + TAG_LENGTH; + return padded ? -1 : IV_LENGTH + TAG_LENGTH; + } + + @Override + public int getEncryptedPageBodySize(int plaintextSize) { + return padded + ? 32 + (plaintextSize / 16 + 1) * 16 + : IEncrypt.super.getEncryptedPageBodySize(plaintextSize); + } + + @Override + public int getPagePlaintextSize(java.nio.ByteBuffer body) { + if (!padded) { + return IEncrypt.super.getPagePlaintextSize(body); + } + if (body.remaining() < 48) { + throw new EncryptException("Truncated test padded page"); + } + return body.getInt(); } @Override @@ -206,9 +234,15 @@ public void close() { private static final class AeadEncryptor implements IEncryptor { private final byte[] dataKey; + private final boolean padded; private AeadEncryptor(byte[] dataKey) { + this(dataKey, false); + } + + private AeadEncryptor(byte[] dataKey, boolean padded) { this.dataKey = dataKey; + this.padded = padded; } @Override @@ -224,7 +258,14 @@ public byte[] encrypt(byte[] data, int offset, int size) { @Override public byte[] encryptPage( byte[] data, int offset, int size, PageCryptoContext pageCryptoContext) { - return encryptWithAad(data, offset, size, pageCryptoContext.getAssociatedData()); + if (!padded) { + return encryptWithAad(data, offset, size, pageCryptoContext.getAssociatedData()); + } + byte[] input = new byte[(size / 16 + 1) * 16]; + System.arraycopy(data, offset, input, 0, size); + byte[] encrypted = + encryptWithAad(input, 0, input.length, pageCryptoContext.getAssociatedData()); + return java.nio.ByteBuffer.allocate(4 + encrypted.length).putInt(size).put(encrypted).array(); } private byte[] encryptWithAad(byte[] data, int offset, int size, byte[] associatedData) { @@ -256,9 +297,15 @@ public EncryptionType getEncryptionType() { private static final class AeadDecryptor implements IDecryptor { private final byte[] dataKey; + private final boolean padded; private AeadDecryptor(byte[] dataKey) { + this(dataKey, false); + } + + private AeadDecryptor(byte[] dataKey, boolean padded) { this.dataKey = dataKey; + this.padded = padded; } @Override @@ -274,7 +321,12 @@ public byte[] decrypt(byte[] data, int offset, int size) { @Override public byte[] decryptPage( byte[] data, int offset, int size, PageCryptoContext pageCryptoContext) { - return decryptWithAad(data, offset, size, pageCryptoContext.getAssociatedData()); + if (!padded) { + return decryptWithAad(data, offset, size, pageCryptoContext.getAssociatedData()); + } + byte[] decoded = + decryptWithAad(data, offset + 4, size - 4, pageCryptoContext.getAssociatedData()); + return Arrays.copyOf(decoded, pageCryptoContext.getCompressedPlaintextSize()); } private byte[] decryptWithAad(byte[] data, int offset, int size, byte[] associatedData) { diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/PageAeadReaderLifecycleTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/PageAeadReaderLifecycleTest.java index ecfb52d03..1d2a497a3 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/PageAeadReaderLifecycleTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/PageAeadReaderLifecycleTest.java @@ -103,6 +103,21 @@ public void testPlaintextQueriesAreNotStronglyRetained() throws Exception { } } + @Test + public void testPaddedBodiesInAlignedAndNonAlignedQueries() throws Exception { + for (boolean aligned : new boolean[] {false, true}) { + File file = writeFile(aligned, true); + try (TsFileReader reader = new TsFileReader(file)) { + QueryDataSet query = reader.query(queryExpression()); + int count = 0; + while (query.hasNext()) { + assertEquals(++count * 10L, query.next().getFields().get(0).getLongV()); + } + assertEquals(3, count); + } + } + } + @Test public void testSeriesScanReleasesEachChunkAndCreatesOneProvider() throws Exception { File file = writeFile(false); @@ -384,10 +399,16 @@ private static TsFileSequenceReader metadataOffsetReader( } private File writeFile(boolean aligned) throws Exception { - File file = new File(temporaryFolder.getRoot(), "lifecycle.tsfile"); + return writeFile(aligned, false); + } + + private File writeFile(boolean aligned, boolean padded) throws Exception { + File file = + new File(temporaryFolder.getRoot(), "lifecycle-" + aligned + "-" + padded + ".tsfile"); try (EncryptParameter parameter = - TestAeadEncryptionProvider.createParameter( - new byte[16], new byte[EncryptParameter.FILE_CRYPTO_ID_LENGTH]); + padded + ? TestAeadEncryptionProvider.createPaddedParameter(new byte[16], new byte[16]) + : TestAeadEncryptionProvider.createParameter(new byte[16], new byte[16]); TsFileWriter writer = new TsFileWriter(file, parameter)) { MeasurementSchema first = new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.PLAIN); if (aligned) { diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/page/PageBodyEncoderTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/page/PageBodyEncoderTest.java index 33d0e0bfd..60f5e8a7f 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/write/page/PageBodyEncoderTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/write/page/PageBodyEncoderTest.java @@ -92,19 +92,26 @@ public void testCompressedBodyUsesOnlySliceAndValidOutputBytes() throws Exceptio @Test public void testEncryptedBodyRoundTripForSlicedInput() throws Exception { + encryptedBodyRoundTrip(false); + encryptedBodyRoundTrip(true); + } + + private void encryptedBodyRoundTrip(boolean padded) throws Exception { for (CompressionType compression : new CompressionType[] { CompressionType.UNCOMPRESSED, CompressionType.GZIP, CompressionType.LZ4 }) { ByteBuffer input = slicedInput(); try (EncryptParameter parameter = - TestAeadEncryptionProvider.createParameter(new byte[16], new byte[16])) { + padded + ? TestAeadEncryptionProvider.createPaddedParameter(new byte[16], new byte[16]) + : TestAeadEncryptionProvider.createParameter(new byte[16], new byte[16])) { ByteBuffer encoded = PageBodyEncoder.encode( input, input.remaining(), ICompressor.getCompressor(compression), parameter, 3, 7); PageCryptoContext context = PageCryptoContext.forDecryption( - parameter, input.remaining(), encoded.remaining(), 3, 7); + parameter, input.remaining(), encoded.asReadOnlyBuffer(), 3, 7); byte[] compressed = IDecryptor.getDecryptor(parameter) .decryptPage(