From b8b83b5078f3d6467dedb07192e83e66f71f102d Mon Sep 17 00:00:00 2001 From: Zihan Dai Date: Tue, 3 Nov 2026 16:19:47 +1100 Subject: [PATCH] Mark the casters of TSDataType transient Since #872, every TSDataType constant holds a SingleValueCaster and an ArrayCaster, initialised with lambdas that are not serializable. Java serializes an enum constant only by its name, but Flink's ClosureCleaner checks every non-static, non-transient field of the objects reachable from a function, including the fields of enum constants. A Flink job whose function or input format reaches a TSDataType through its fields is therefore rejected with "TSDataType$$Lambda... is not serializable". Mark both fields transient. Nothing reads them after serialization, since deserializing an enum constant returns the existing constant. --- .../org/apache/tsfile/enums/TSDataType.java | 7 +- .../enums/TSDataTypeSerializationTest.java | 72 +++++++++++++++++++ 2 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 java/common/src/test/java/org/apache/tsfile/enums/TSDataTypeSerializationTest.java diff --git a/java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java b/java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java index eaa09be738..71a4311080 100644 --- a/java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java +++ b/java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java @@ -126,8 +126,11 @@ public enum TSDataType { private final int dataTypeSize; private final boolean numeric; private final boolean comparable; - private final SingleValueCaster singleValueCaster; - private final ArrayCaster arrayCaster; + // Java serializes an enum constant by its name only, so these never need to be serialized. + // They are transient because some tools, such as Flink's ClosureCleaner, still check every + // non-transient field and reject a function that holds a TSDataType. + private final transient SingleValueCaster singleValueCaster; + private final transient ArrayCaster arrayCaster; private static final Map> compatibleTypes; static { diff --git a/java/common/src/test/java/org/apache/tsfile/enums/TSDataTypeSerializationTest.java b/java/common/src/test/java/org/apache/tsfile/enums/TSDataTypeSerializationTest.java new file mode 100644 index 0000000000..c319a0ec40 --- /dev/null +++ b/java/common/src/test/java/org/apache/tsfile/enums/TSDataTypeSerializationTest.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.enums; + +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; + +import static org.junit.Assert.assertSame; + +public class TSDataTypeSerializationTest { + + /** + * Flink's ClosureCleaner serializes every non-static, non-transient field of an object that a + * user function holds, including the fields of an enum constant. A field that is not serializable + * makes it reject the function, although Java serializes the enum constant only by its name. + */ + @Test + public void testEveryCheckedFieldIsSerializable() throws Exception { + for (TSDataType dataType : TSDataType.values()) { + for (Field field : TSDataType.class.getDeclaredFields()) { + int modifiers = field.getModifiers(); + if (Modifier.isStatic(modifiers) || Modifier.isTransient(modifiers)) { + continue; + } + field.setAccessible(true); + serialize(field.get(dataType)); + } + } + } + + @Test + public void testJavaSerializationKeepsTheConstant() throws Exception { + for (TSDataType dataType : TSDataType.values()) { + try (ObjectInputStream in = + new ObjectInputStream(new ByteArrayInputStream(serialize(dataType)))) { + assertSame(dataType, in.readObject()); + } + } + } + + private static byte[] serialize(Object value) throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (ObjectOutputStream out = new ObjectOutputStream(bytes)) { + out.writeObject(value); + } + return bytes.toByteArray(); + } +}