Derive lossless NOT NULL relaxation from durable table declarations

src/agent_comms/typed_table.py

@@ -660,6 +660,14 @@ def differences(
                      if stored.get(name) != expected.get(name))
 
 
+@dataclass(frozen=True)
+class SQLiteColumnNullability(TypedRow):
+    """SQLite's column names and NOT NULL declarations, read without parsing DDL."""
+
+    name: str
+    notnull: bool
+
+
 @dataclass(frozen=True)
 class SQLiteTableName(TypedRow):
     """One table SQLite's catalog holds, whether or not a row declaration names it."""
@@ -878,6 +886,7 @@ def derive_schema(
         for row in cls.schema_rows():
             row.take_previous_name(db)
             row.derive_nullable_fields(db)
+            row.derive_relaxed_fields(db)
         SQLiteCatalogObject.carry(db, cls.declared_schema(),
                                   frozenset(item.name for item in cls.stored_schema(db, sharing)))
         cls.require_schema(db, sharing)
@@ -1593,7 +1602,10 @@ def _sql(cls, expression: str) -> str:
         return expression
 
     @classmethod
-    def _table_sql(cls, *, if_not_exists: bool = False, columns: tuple[str, ...] = ()) -> str:
+    def _table_sql(
+        cls, *, if_not_exists: bool = False, columns: tuple[str, ...] = (),
+        required: frozenset[str] = frozenset(),
+    ) -> str:
         existence = "IF NOT EXISTS " if if_not_exists else ""
         definitions = []
         fields = cls._fields_named(columns) if columns else cls._columns
@@ -1607,7 +1619,7 @@ def _table_sql(cls, *, if_not_exists: bool = False, columns: tuple[str, ...] = (
             if column.auto_increment:
                 sql += " PRIMARY KEY AUTOINCREMENT"
             sql += cls._sql(column.generated_sql)
-            if not item.nullable or column.primary_key:
+            if not item.nullable or column.primary_key or item.name in required:
                 sql += " NOT NULL"
             if column.unique:
                 sql += " UNIQUE"
@@ -1727,6 +1739,32 @@ def derive_nullable_fields(cls, db: sqlite3.Connection) -> None:
         if actual.get(cls.declared_name) == previous:
             cls.rebuild(db)
 
+    @classmethod
+    def derive_relaxed_fields(cls, db: sqlite3.Connection) -> None:
+        """Only an exact earlier declaration requiring now-nullable fields can relax rows.
+
+        Column additions, removals and other table differences still require a data conversion.
+        Compare through the same DDL owner; only writable non-key fields may widen.
+        """
+        stored = SQLiteColumnNullability.read(db.execute(
+            'SELECT name, "notnull" FROM pragma_table_info(?)', (cls.declared_name,),
+        ))
+        if tuple(column.name for column in stored) != tuple(
+            item.wire_name for item in cls._columns
+        ):
+            return
+        nonnull = frozenset(column.name for column in stored if column.notnull)
+        required = frozenset(
+            item.name for item in cls._columns
+            if item.nullable and item.column.writable and not item.column.primary_key
+            and item.wire_name in nonnull
+        )
+        if not required:
+            return
+        actual = {item.name: item.sql for item in cls.stored_objects(db)}
+        if actual.get(cls.declared_name) == cls._table_sql(required=required):
+            cls.rebuild(db)
+
     @classmethod
     def take_previous_name(cls, db: sqlite3.Connection) -> None:
         """Rows stored under a declared earlier name move to the declared one; rows under both

tests/round2/test_typed_table.py

@@ -156,6 +156,32 @@ def test_writer_derives_missing_nullable_fields_but_refuses_unclaimed_data():
         assert db.execute(f'SELECT unclaimed FROM "{GoalHistoryEntry.declared_name}"').fetchall() == [('preserve this',)]
 
 
+def test_durable_writer_relaxes_not_null_without_losing_rows():
+    from agent_comms.typed_table import DurableTables, WriterOwnedFile
+
+    @dataclass(frozen=True)
+    class RelaxedValueRow(DurableTables, TypedTable, declared_name="relaxed_values"):
+        key: Annotated[str, Column(primary_key=True)]
+        value: str | None
+
+    rows = [RelaxedValueRow("first", "one"), RelaxedValueRow("second", "two")]
+    with sqlite3.connect(":memory:") as db:
+        db.execute(
+            'CREATE TABLE "relaxed_values" ("key" TEXT NOT NULL, "value" TEXT NOT NULL, '
+            'PRIMARY KEY ("key")) STRICT'
+        )
+        for row in rows:
+            row.insert(db)
+        db.commit()
+
+        RelaxedValueRow.reconcile_schema(db, WriterOwnedFile())
+
+        assert RelaxedValueRow.select(db, order_by=("key",)) == rows
+        RelaxedValueRow("empty", None).insert(db)
+        assert RelaxedValueRow.require(db, key="empty").value is None
+        assert RelaxedValueRow.count(db) == len(rows) + 1
+
+
 def test_atomic_creation_preserves_declared_indexes_triggers_and_strict_creation():
     from agent_comms.native_pi import NativeContextJournal