From 2c106e08808fa1dc65be4f3b6309856a0e57981a Mon Sep 17 00:00:00 2001 From: Adam Rauch Date: Thu, 30 Jul 2026 07:45:18 -0700 Subject: [PATCH 1/3] Stream StudyCachables in QueryHelper --- api/src/org/labkey/api/data/Selector.java | 2 +- .../org/labkey/api/study/QueryHelper.java | 30 +++++++++++-------- .../org/labkey/study/model/StudyManager.java | 21 ++++++++----- 3 files changed, 31 insertions(+), 22 deletions(-) diff --git a/api/src/org/labkey/api/data/Selector.java b/api/src/org/labkey/api/data/Selector.java index 3ac7d2256e3..f4ba4f30c5d 100644 --- a/api/src/org/labkey/api/data/Selector.java +++ b/api/src/org/labkey/api/data/Selector.java @@ -77,7 +77,7 @@ public interface Selector /** * Returns a sequential Stream of objects or records representing rows from the database. Converts each result row - * into an object the specified {@code Class}. The Stream is backed by a cached data structure (ResultSet and + * into an object of the specified {@code Class}. The Stream is backed by a cached data structure (ResultSet and * Connection are closed before returning the stream), so no need to close or fully exhaust this stream. Cached * streams are more convenient to use than uncached streams and should perform well in low-volume situations. */ diff --git a/study/api-src/org/labkey/api/study/QueryHelper.java b/study/api-src/org/labkey/api/study/QueryHelper.java index 50fa0186059..4db3acd3624 100644 --- a/study/api-src/org/labkey/api/study/QueryHelper.java +++ b/study/api-src/org/labkey/api/study/QueryHelper.java @@ -29,12 +29,14 @@ import org.labkey.api.data.TableInfoGetter; import org.labkey.api.data.TableSelector; import org.labkey.api.security.User; +import org.labkey.api.study.QueryHelper.StudyCacheCollections; import java.util.Collection; import java.util.Collections; import java.util.Map; +import java.util.stream.Stream; -public class QueryHelper, SC extends QueryHelper.StudyCacheCollections> +public class QueryHelper, SC extends StudyCacheCollections> { private final BlockingCache _cache; private final Class _objectClass; @@ -52,12 +54,13 @@ public QueryHelper(TableInfoGetter tableInfoGetter, Class objectClass, @Nulla _objectClass = objectClass; _defaultSortString = defaultSortString; TableInfo tableInfo = _tableInfoGetter.getTableInfo(); - _cache = DatabaseCache.get(tableInfo.getSchema().getScope(), tableInfo.getCacheSize(), "StudyCache: " + tableInfo.getName(), (key, argument) -> - createCollections( - getTableSelector(key).stream(_objectClass) - .peek(StudyCachable::lock) - .toList() - )); + _cache = DatabaseCache.get(tableInfo.getSchema().getScope(), tableInfo.getCacheSize(), "StudyCache: " + tableInfo.getName(), (key, _) -> + { + try (Stream stream = getTableSelector(key).uncachedStream(_objectClass)) + { + return createCollections(stream); + } + }); } /** @@ -83,9 +86,9 @@ protected SC getCollections(Container c) return _cache.get(c, null); } - protected SC createCollections(Collection collection) + protected SC createCollections(Stream stream) { - return (SC) new QueryHelper.StudyCacheCollections<>(collection); + return (SC) new StudyCacheCollections<>(stream); } public T create(User user, T obj) @@ -133,11 +136,12 @@ public static class StudyCacheCollections> { private final Map _map; - // Receives a collection of locked T objects - public StudyCacheCollections(Collection collection) + public StudyCacheCollections(Stream stream) { - _map = Collections.unmodifiableMap(collection.stream() - .collect(LabKeyCollectors.toLinkedMap(StudyCachable::getPrimaryKey, v -> v))); + _map = Collections.unmodifiableMap(stream + .peek(StudyCachable::lock) + .collect(LabKeyCollectors.toLinkedMap(StudyCachable::getPrimaryKey, v -> v)) + ); } public @Nullable V get(K key) diff --git a/study/src/org/labkey/study/model/StudyManager.java b/study/src/org/labkey/study/model/StudyManager.java index 677d8135613..add84ec1ab4 100644 --- a/study/src/org/labkey/study/model/StudyManager.java +++ b/study/src/org/labkey/study/model/StudyManager.java @@ -224,6 +224,7 @@ import java.util.WeakHashMap; import java.util.function.Consumer; import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.labkey.api.action.SpringActionController.ERROR_MSG; import static org.labkey.api.studydesign.query.StudyDesignQuerySchema.PERSONNEL_TABLE_NAME; @@ -354,9 +355,9 @@ private Collection getCollection(Container c, Order order) } @Override - protected VisitCollections createCollections(Collection collection) + protected VisitCollections createCollections(Stream stream) { - return new VisitCollections(collection); + return new VisitCollections(stream); } private static class VisitCollections extends StudyCacheCollections @@ -364,9 +365,11 @@ private static class VisitCollections extends StudyCacheCollections _sequenceNumVisits; private final Collection _chronologicalVisits; - private VisitCollections(Collection collection) + private VisitCollections(Stream stream) { - super(collection); + super(stream); + + Collection collection = getCollection(); // I'd prefer to push comparators into Visit.Order, but Visit (in API) doesn't know about the display // order field. @@ -427,9 +430,9 @@ public void clearCache(Container c) } @Override - protected DatasetCollections createCollections(Collection collection) + protected DatasetCollections createCollections(Stream stream) { - return new DatasetCollections(collection); + return new DatasetCollections(stream); } protected DatasetCollections getCollections(Study study) @@ -447,9 +450,11 @@ private static class DatasetCollections extends StudyCacheCollections> _cohortMap; private final List _nullCohortDatasets; - private DatasetCollections(Collection collection) + private DatasetCollections(Stream stream) { - super(collection); + super(stream); + + Collection collection = getCollection(); // study.Dataset has constraints on LOWER(Name) and LOWER(Label), so this code path should never attempt // to put duplicates into these maps. Use asserts to verify this. From 0d4cc29c1e1bcfb38c0dfcc488aa154243bddf6e Mon Sep 17 00:00:00 2001 From: Adam Rauch Date: Thu, 30 Jul 2026 08:50:55 -0700 Subject: [PATCH 2/3] Close stream before constructors do their work --- .../org/labkey/api/study/QueryHelper.java | 18 ++++++++++-------- .../org/labkey/study/model/StudyManager.java | 14 +++++++------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/study/api-src/org/labkey/api/study/QueryHelper.java b/study/api-src/org/labkey/api/study/QueryHelper.java index 4db3acd3624..9b3d31b4d92 100644 --- a/study/api-src/org/labkey/api/study/QueryHelper.java +++ b/study/api-src/org/labkey/api/study/QueryHelper.java @@ -58,7 +58,10 @@ public QueryHelper(TableInfoGetter tableInfoGetter, Class objectClass, @Nulla { try (Stream stream = getTableSelector(key).uncachedStream(_objectClass)) { - return createCollections(stream); + return createCollections(Collections.unmodifiableMap(stream + .peek(StudyCachable::lock) + .collect(LabKeyCollectors.toLinkedMap(StudyCachable::getPrimaryKey, v -> v)) + )); } }); } @@ -86,9 +89,10 @@ protected SC getCollections(Container c) return _cache.get(c, null); } - protected SC createCollections(Stream stream) + // map is an unmodifiable, linked map of pk -> locked object + protected SC createCollections(Map map) { - return (SC) new StudyCacheCollections<>(stream); + return (SC) new StudyCacheCollections<>(map); } public T create(User user, T obj) @@ -136,12 +140,10 @@ public static class StudyCacheCollections> { private final Map _map; - public StudyCacheCollections(Stream stream) + // map should be an unmodifiable, linked map of pk -> locked objects + public StudyCacheCollections(Map map) { - _map = Collections.unmodifiableMap(stream - .peek(StudyCachable::lock) - .collect(LabKeyCollectors.toLinkedMap(StudyCachable::getPrimaryKey, v -> v)) - ); + _map = map; } public @Nullable V get(K key) diff --git a/study/src/org/labkey/study/model/StudyManager.java b/study/src/org/labkey/study/model/StudyManager.java index add84ec1ab4..3af1982ade9 100644 --- a/study/src/org/labkey/study/model/StudyManager.java +++ b/study/src/org/labkey/study/model/StudyManager.java @@ -355,9 +355,9 @@ private Collection getCollection(Container c, Order order) } @Override - protected VisitCollections createCollections(Stream stream) + protected VisitCollections createCollections(Map map) { - return new VisitCollections(stream); + return new VisitCollections(map); } private static class VisitCollections extends StudyCacheCollections @@ -365,9 +365,9 @@ private static class VisitCollections extends StudyCacheCollections _sequenceNumVisits; private final Collection _chronologicalVisits; - private VisitCollections(Stream stream) + private VisitCollections(Map map) { - super(stream); + super(map); Collection collection = getCollection(); @@ -430,9 +430,9 @@ public void clearCache(Container c) } @Override - protected DatasetCollections createCollections(Stream stream) + protected DatasetCollections createCollections(Map map) { - return new DatasetCollections(stream); + return new DatasetCollections(map); } protected DatasetCollections getCollections(Study study) @@ -450,7 +450,7 @@ private static class DatasetCollections extends StudyCacheCollections> _cohortMap; private final List _nullCohortDatasets; - private DatasetCollections(Stream stream) + private DatasetCollections(Map stream) { super(stream); From fd9d5a8ddaf026c5a3e91c4670da7544ddc5619a Mon Sep 17 00:00:00 2001 From: Adam Rauch Date: Thu, 30 Jul 2026 10:59:57 -0700 Subject: [PATCH 3/3] This is what I meant... --- study/api-src/org/labkey/api/study/QueryHelper.java | 8 +++++--- study/src/org/labkey/study/model/StudyManager.java | 5 ++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/study/api-src/org/labkey/api/study/QueryHelper.java b/study/api-src/org/labkey/api/study/QueryHelper.java index 9b3d31b4d92..2816726a9f5 100644 --- a/study/api-src/org/labkey/api/study/QueryHelper.java +++ b/study/api-src/org/labkey/api/study/QueryHelper.java @@ -56,13 +56,15 @@ public QueryHelper(TableInfoGetter tableInfoGetter, Class objectClass, @Nulla TableInfo tableInfo = _tableInfoGetter.getTableInfo(); _cache = DatabaseCache.get(tableInfo.getSchema().getScope(), tableInfo.getCacheSize(), "StudyCache: " + tableInfo.getName(), (key, _) -> { + final Map map; try (Stream stream = getTableSelector(key).uncachedStream(_objectClass)) { - return createCollections(Collections.unmodifiableMap(stream + map = Collections.unmodifiableMap(stream .peek(StudyCachable::lock) .collect(LabKeyCollectors.toLinkedMap(StudyCachable::getPrimaryKey, v -> v)) - )); + ); } + return createCollections(map); }); } @@ -141,7 +143,7 @@ public static class StudyCacheCollections> private final Map _map; // map should be an unmodifiable, linked map of pk -> locked objects - public StudyCacheCollections(Map map) + protected StudyCacheCollections(Map map) { _map = map; } diff --git a/study/src/org/labkey/study/model/StudyManager.java b/study/src/org/labkey/study/model/StudyManager.java index 3af1982ade9..bdf0424e68c 100644 --- a/study/src/org/labkey/study/model/StudyManager.java +++ b/study/src/org/labkey/study/model/StudyManager.java @@ -224,7 +224,6 @@ import java.util.WeakHashMap; import java.util.function.Consumer; import java.util.stream.Collectors; -import java.util.stream.Stream; import static org.labkey.api.action.SpringActionController.ERROR_MSG; import static org.labkey.api.studydesign.query.StudyDesignQuerySchema.PERSONNEL_TABLE_NAME; @@ -450,9 +449,9 @@ private static class DatasetCollections extends StudyCacheCollections> _cohortMap; private final List _nullCohortDatasets; - private DatasetCollections(Map stream) + private DatasetCollections(Map map) { - super(stream); + super(map); Collection collection = getCollection();