add list/array support for duckdb and postgresql

docs/content/_index.md

@@ -59,6 +59,7 @@ Welcome to the sqlgen documentation. This guide provides detailed information ab
   {{< card link="dynamic" title="sqlgen::Dynamic" icon="book-open" subtitle="How to define custom SQL types not natively supported by sqlgen.">}}
   {{< card link="foreign_key" title="sqlgen::ForeignKey" icon="book-open" subtitle="How to establish referential integrity between tables.">}}
   {{< card link="json" title="sqlgen::JSON" icon="book-open" subtitle="How to store and work with JSON fields.">}}
+  {{< card link="list" title="std::vector" icon="book-open" subtitle="How to store std::vector fields as native lists and arrays (DuckDB, PostgreSQL).">}}
   {{< card link="pattern" title="sqlgen::Pattern" icon="book-open" subtitle="How to add regex pattern validation to avoid SQL injection.">}}
   {{< card link="timestamp" title="sqlgen::Timestamp" icon="book-open" subtitle="How timestamps work in sqlgen.">}}
   {{< card link="unique" title="sqlgen::Unique" icon="book-open" subtitle="How to enforce uniqueness constraints on table columns.">}}

docs/content/list.md

@@ -0,0 +1,108 @@
+# `std::vector`
+
+`std::vector<T>` fields map directly to native collection types in supported databases.
+
+Currently supported for DuckDB and PostgreSQL. On other databases, `std::vector` fields fail with a compile-time error.
+
+## Usage
+
+### Basic Definition
+
+Define a list field directly as `std::vector<T>`:
+
+```cpp
+struct ListTable {
+    int32_t id;
+    std::vector<int32_t> numbers;
+    std::vector<std::string> labels;
+};
+```
+
+This generates the following SQL schema:
+
+```sql
+-- DuckDB/PostgreSQL
+CREATE TABLE IF NOT EXISTS "ListTable"("id" INTEGER NOT NULL, "numbers" INTEGER[] NOT NULL, "labels" TEXT[] NOT NULL);
+```
+
+### Nested Lists
+
+Nested vectors are supported to arbitrary depth, with one `[]` suffix added per nesting level:
+
+```cpp
+struct Matrix {
+    int32_t id;
+    std::vector<std::vector<int32_t>> nested;
+};
+```
+
+```sql
+-- DuckDB/PostgreSQL
+CREATE TABLE IF NOT EXISTS "Matrix"("id" INTEGER NOT NULL, "nested" INTEGER[][] NOT NULL);
+```
+
+### Nullable Lists and Nullable Elements
+
+Nullability is controlled the same way as for scalar fields:
+
+- `std::optional<std::vector<T>>`: the entire list may be `NULL`
+- `std::vector<std::optional<T>>`: the list is `NOT NULL`, but individual elements may be `NULL`
+
+```cpp
+struct NullableLists {
+    int32_t id;
+    std::optional<std::vector<int32_t>> nullable_list;
+    std::vector<std::optional<int32_t>> nullable_elements;
+};
+```
+
+```sql
+-- DuckDB / PostgreSQL
+CREATE TABLE IF NOT EXISTS "NullableLists"("id" INTEGER NOT NULL, "nullable_list" INTEGER[], "nullable_elements" INTEGER[] NOT NULL);
+```
+
+### Reading and Writing
+
+Use the regular `sqlgen::write` and `sqlgen::read` APIs.
+
+```cpp
+using namespace sqlgen;
+using namespace sqlgen::literals;
+
+struct Person {
+    int32_t id;
+    std::string first_name;
+    std::vector<std::string> children;
+};
+
+const auto people = std::vector<Person>({
+    Person{.id = 1, .first_name = "Homer", .children = {"Bart", "Lisa", "Maggie"}},
+    Person{.id = 2, .first_name = "Bart", .children = {}}
+});
+
+// DuckDB example (analogous for PostgreSQL)
+const std::vector<Person> result = sqlgen::duckdb::connect()
+    .and_then(create_table<Person>)
+    .and_then(write(std::ref(people)))
+    .and_then(sqlgen::read<std::vector<Person>>)
+    .value();
+
+// result is identical to people
+```
+
+Empty lists and missing values round-trip correctly.
+
+### Database Integration
+
+`std::vector<T>` maps to the dialect's native collection type:
+
+- **DuckDB**: `LIST` (variable-length, no shape restrictions)
+- **PostgreSQL**: `ARRAY` (governed by a strict structural rule)
+
+**Important**: PostgreSQL arrays must be rectangular, so every sub-array within a dimension must have the same length: `{{1, 2}, {3, 4}}` is valid, while `{{1, 2}, {3}}` is not ([PostgreSQL docs](https://www.postgresql.org/docs/current/arrays.html#ARRAYS-INPUT)). sqlgen cannot check this at compile-time, so a non-rectangular value only surfaces as an error when the database rejects it on write.
+
+## Notes
+
+- Only supported for DuckDB and PostgreSQL. Compilation fails with `"std::vector fields are not supported by this database."` if the database doesn't support it.
+- Works with type-safe operations such as `create_table`, `write`/`insert` and `read`
+- Nested vectors (`std::vector<std::vector<T>>`) supported to arbitrary depth

include/sqlgen/create_table.hpp

@@ -4,6 +4,7 @@
 #include <rfl.hpp>
 
 #include "is_connection.hpp"
+#include "internal/supports_collections.hpp"
 #include "transpilation/to_create_table.hpp"
 
 namespace sqlgen {
@@ -12,6 +13,7 @@ template <class ValueType, class Connection>
   requires is_connection<Connection>
 Result<Ref<Connection>> create_table_impl(const Ref<Connection>& _conn,
                                           const bool _if_not_exists) {
+  internal::assert_collections_supported<Connection, ValueType>();
   const auto query = transpilation::to_create_table<ValueType>(_if_not_exists);
   return _conn->execute(_conn->to_sql(query)).transform([&](const auto&) {
     return _conn;

include/sqlgen/duckdb/Connection.hpp

@@ -22,6 +22,7 @@
 #include "../dynamic/Write.hpp"
 #include "../internal/iterator_t.hpp"
 #include "../internal/remove_auto_incr_primary_t.hpp"
+#include "../internal/supports_collections.hpp"
 #include "../internal/to_container.hpp"
 #include "../is_connection.hpp"
 #include "../sqlgen_api.hpp"
@@ -31,6 +32,7 @@
 #include "DuckDBResult.hpp"
 #include "Iterator.hpp"
 #include "to_sql.hpp"
+#include "write_data_chunk.hpp"
 
 namespace sqlgen::duckdb {
 
@@ -156,16 +158,74 @@ class SQLGEN_API Connection {
   template <class ItBegin, class ItEnd>
   Result<Nothing> write_to_appender(ItBegin _begin, ItEnd _end,
                                     duckdb_appender _appender) {
-    for (auto it = _begin; it < _end; ++it) {
-      const auto res = write_row(*it, _appender);
-      if (!res) {
-        return res;
+    using ValueType =
+        std::remove_cvref_t<typename std::iterator_traits<ItBegin>::value_type>;
+    if constexpr (chunk_writer::has_vector_field_v<ValueType>) {
+      return write_data_chunks(_begin, _end, _appender);
+    } else {
+      for (auto it = _begin; it < _end; ++it) {
+        const auto res = write_row(*it, _appender);
+        if (!res) {
+          return res;
+        }
+        const auto state = duckdb_appender_end_row(_appender);
+        if (state == DuckDBError) {
+          return error(duckdb_appender_error(_appender));
+        }
       }
-      const auto state = duckdb_appender_end_row(_appender);
-      if (state == DuckDBError) {
-        return error(duckdb_appender_error(_appender));
+      return Nothing{};
+    }
+  }
+
+  template <class ItBegin, class ItEnd>
+  Result<Nothing> write_data_chunks(ItBegin _begin, ItEnd _end,
+                                    duckdb_appender _appender) {
+    const auto column_count = duckdb_appender_column_count(_appender);
+    std::vector<duckdb_logical_type> types(column_count);
+    for (idx_t i = 0; i < column_count; ++i) {
+      types[i] = duckdb_appender_column_type(_appender, i);
+      if (!types[i]) {
+        for (idx_t j = 0; j < i; ++j) {
+          duckdb_destroy_logical_type(&types[j]);
+        }
+        return error("Could not retrieve DuckDB appender column type.");
       }
     }
+
+    auto destroy_types = [&] {
+      for (auto& type : types) duckdb_destroy_logical_type(&type);
+    };
+
+    auto it = _begin;
+    while (it != _end) {
+      auto chunk = duckdb_create_data_chunk(types.data(), column_count);
+      if (!chunk) {
+        destroy_types();
+        return error("Could not create DuckDB data chunk.");
+      }
+
+      idx_t row = 0;
+      for (; it != _end && row < duckdb_vector_size(); ++it, ++row) {
+        auto result = chunk_writer::assign_row(chunk, row, *it);
+        if (!result) {
+          duckdb_destroy_data_chunk(&chunk);
+          destroy_types();
+          return result;
+        }
+      }
+      duckdb_data_chunk_set_size(chunk, row);
+      if (duckdb_append_data_chunk(_appender, chunk) == DuckDBError) {
+        const auto appender_error = duckdb_appender_error(_appender);
+        const auto message =
+            appender_error ? std::string(appender_error)
+                           : std::string("Could not append DuckDB data chunk.");
+        duckdb_destroy_data_chunk(&chunk);
+        destroy_types();
+        return error(message);
+      }
+      duckdb_destroy_data_chunk(&chunk);
+    }
+    destroy_types();
     return Nothing{};
   }
 
@@ -203,6 +263,9 @@ struct IteratorType<ValueType, duckdb::Connection> {
   using Type = duckdb::Iterator<ValueType>;
 };
 
+template <>
+struct supports_collections<duckdb::Connection> : std::true_type {};
+
 static_assert(is_connection<duckdb::Connection>,
               "Must fulfill the is_connection concept.");
 static_assert(is_connection<Session<duckdb::Connection>>,

include/sqlgen/duckdb/make_chunk_ptrs.hpp

@@ -10,11 +10,13 @@
 #include <utility>
 
 #include "../Ref.hpp"
+#include "../transpilation/is_vector.hpp"
 #include "ColumnData.hpp"
 #include "DuckDBResult.hpp"
 #include "cast_duckdb_type.hpp"
 #include "check_duckdb_type.hpp"
 #include "chunk_ptrs_t.hpp"
+#include "read_list_vector.hpp"
 
 namespace sqlgen::duckdb {
 
@@ -42,7 +44,25 @@ struct MakeChunkPtrs<rfl::Tuple<ColumnData<Ts, ColNames>...>> {
 
     auto vec = duckdb_data_chunk_get_vector(_chunk, _i);
 
-    if (check_duckdb_type<T>(actual_duckdb_type)) {
+    if constexpr (transpilation::is_vector_v<T>) {
+      if (actual_duckdb_type != DUCKDB_TYPE_LIST ||
+          !check_list_vector_type<T>(vec)) {
+        throw std::runtime_error("Wrong type in field '" + ColName().str() +
+                                 "'. Expected a DuckDB LIST.");
+      }
+      auto values =
+          Ref<std::vector<T>>::make(duckdb_data_chunk_get_size(_chunk));
+      const auto validity = duckdb_vector_get_validity(vec);
+      for (idx_t i = 0; i < values->size(); ++i) {
+        if (!validity || duckdb_validity_row_is_valid(validity, i)) {
+          (*values)[i] = read_list_vector_value<T>(vec, i).value();
+        }
+      }
+      return ColumnData<T, ColName>{.vec = vec,
+                                    .data = values->data(),
+                                    .validity = validity,
+                                    .ptr = values.ptr()};
+    } else if (check_duckdb_type<T>(actual_duckdb_type)) {
       return ColumnData<T, ColName>{
           .vec = vec,
           .data = static_cast<T*>(duckdb_vector_get_data(vec)),

include/sqlgen/duckdb/parsing/Parser.hpp

@@ -11,5 +11,6 @@
 #include "Parser_smart_ptr.hpp"
 #include "Parser_string.hpp"
 #include "Parser_timestamp.hpp"
+#include "Parser_vector.hpp"
 
 #endif

include/sqlgen/duckdb/parsing/Parser_date.hpp

@@ -35,6 +35,14 @@ struct Parser<Date> {
                ? Result<Nothing>(Nothing{})
                : Result<Nothing>(error("Could not append date value."));
   }
+
+  static Result<Nothing> write_to_vector(duckdb_vector _vector, idx_t _index,
+                                         const Date& _value) {
+    static_cast<duckdb_date*>(duckdb_vector_get_data(_vector))[_index] =
+        duckdb_date{
+            .days = static_cast<int32_t>(_value.to_time_t() / seconds_per_day)};
+    return Nothing{};
+  }
 };
 
 }  // namespace sqlgen::duckdb::parsing

include/sqlgen/duckdb/parsing/Parser_default.hpp

@@ -87,6 +87,19 @@ struct Parser {
       return error("Unsupported type.");
     }
   }
+
+  static Result<Nothing> write_to_vector(duckdb_vector _vector, idx_t _index,
+                                         const T& _value) {
+    if constexpr (std::is_arithmetic_v<Type>) {
+      static_cast<Type*>(duckdb_vector_get_data(_vector))[_index] = _value;
+      return Nothing{};
+
+    } else {
+      static_assert(rfl::always_false_v<T>,
+                    "Unsupported DuckDB data chunk value type.");
+      return error("Unsupported DuckDB data chunk value type.");
+    }
+  }
 };
 
 }  // namespace sqlgen::duckdb::parsing

include/sqlgen/duckdb/parsing/Parser_enum.hpp

@@ -37,6 +37,13 @@ struct Parser<EnumT> {
                ? Result<Nothing>(Nothing{})
                : Result<Nothing>(error("Could not append string value."));
   }
+
+  static Result<Nothing> write_to_vector(duckdb_vector _vector, idx_t _index,
+                                         const EnumT& _value) {
+    static_cast<uint8_t*>(duckdb_vector_get_data(_vector))[_index] =
+        static_cast<uint8_t>(_value);
+    return Nothing{};
+  }
 };
 
 }  // namespace sqlgen::duckdb::parsing

include/sqlgen/duckdb/parsing/Parser_json.hpp

@@ -32,6 +32,16 @@ struct Parser<JSON<T>> {
       return error(e.what());
     }
   }
+
+  static Result<Nothing> write_to_vector(duckdb_vector _vector, idx_t _index,
+                                         const JSON<T>& _value) {
+    try {
+      return Parser<std::string>::write_to_vector(
+          _vector, _index, rfl::json::write(_value.value()));
+    } catch (const std::exception& e) {
+      return error(e.what());
+    }
+  }
 };
 
 }  // namespace sqlgen::duckdb::parsing

include/sqlgen/duckdb/parsing/Parser_optional.hpp

@@ -8,6 +8,7 @@
 #include <type_traits>
 
 #include "../../Result.hpp"
+#include "../set_null.hpp"
 #include "Parser_base.hpp"
 
 namespace sqlgen::duckdb::parsing {
@@ -36,6 +37,16 @@ struct Parser<std::optional<T>> {
     }
     return Parser<std::remove_cvref_t<T>>::write(*_o, _appender);
   }
+
+  static Result<Nothing> write_to_vector(duckdb_vector _vector, idx_t _index,
+                                         const std::optional<T>& _value) {
+    if (!_value) {
+      set_null(_vector, _index);
+      return Nothing{};
+    }
+    return Parser<std::remove_cvref_t<T>>::write_to_vector(_vector, _index,
+                                                           *_value);
+  }
 };
 
 }  // namespace sqlgen::duckdb::parsing

include/sqlgen/duckdb/parsing/Parser_reflection_type.hpp

@@ -29,6 +29,12 @@ struct Parser<T> {
     return Parser<std::remove_cvref_t<typename Type::ReflectionType>>::write(
         _t.reflection(), _appender);
   }
+
+  static Result<Nothing> write_to_vector(duckdb_vector _vector, idx_t _index,
+                                         const T& _value) {
+    return Parser<std::remove_cvref_t<typename Type::ReflectionType>>::
+        write_to_vector(_vector, _index, _value.reflection());
+  }
 };
 
 }  // namespace sqlgen::duckdb::parsing