test: exercise Codex backend through executable Port runner

mix.lock

@@ -7,7 +7,7 @@
   "erlex": {:hex, :erlex, "0.2.9", "7debbbaa9f4f368b8cd648983e0f1d7963028508e9c59e9d4ed504e94ef52a55", [:mix], [], "hexpm", "8cfffc0ec7159e6d73de2ab28a588064de80f88b2798d5cbe4482cbbc200178b"},
   "ex_doc": {:hex, :ex_doc, "0.40.4", "66f2e42bf588594d5a8aab31cad87f2ddad09d0da1b1a2f379340ec2c2e497cb", [:mix], [{:earmark_parser, "~> 1.4.46", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "6222b9e423d76584ee34df2c82a5ed72c2d53dc153f7f483ad28b378694186cc"},
   "file_system": {:hex, :file_system, "1.1.1", "31864f4685b0148f25bd3fbef2b1228457c0c89024ad67f7a81a3ffbc0bbad3a", [:mix], [], "hexpm", "7a15ff97dfe526aeefb090a7a9d3d03aa907e100e262a0f8f7746b78f8f87a5d"},
-  "gen_agent": {:hex, :gen_agent, "0.2.0", "22d16e4124a5913882a2a8855cfdf0d9c696ae0a13afc131cd0919613f8e76ec", [:mix], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9c2474e32f81c81a9e1787dabcec93c78ca20520d7b9559184404e58b43f38e2"},
+  "gen_agent": {:hex, :gen_agent, "0.2.1", "b55b3e24df8b9693763a0c34ade823b4bd92ed5c90e8ca64e5fb6e23e6eae973", [:mix], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "6355ca87301567f0f6aee98bef61de797ccd8cc0e1f93fbb81bece7ef8474dde"},
   "jason": {:hex, :jason, "1.4.5", "2e3a008590b0b8d7388c20293e9dcc9cf3e5d642fd2a114e4cbbb52e595d940a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "b0c823996102bcd0239b3c2444eb00409b72f6a140c1950bc8b457d836b30684"},
   "makeup": {:hex, :makeup, "1.2.2", "882d46dc0905e9ff7abf2aab61a7e6b3dcc555533977d8a23b06019e6c89ac94", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "9a1a24e5b343b8ae16abea0822c10a6f75da27af7fa802ada5251f7579bfccfa"},
   "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"},

test/fixtures/codex_cli.sh

@@ -0,0 +1,52 @@
+#!/bin/sh
+set -eu
+
+fixture_dir=$(cd "$(dirname "$0")" && pwd)
+
+mode=fresh
+if [ "${2:-}" = "resume" ]; then
+  mode=resume
+fi
+
+printf '%s\n' "$@" > "$fixture_dir/$mode.args"
+
+case "${*}" in
+  *fail*)
+    fixture_mode=failure
+    ;;
+  *hold*)
+    fixture_mode=hold
+    ;;
+  *)
+    fixture_mode=success
+    ;;
+esac
+
+case "$fixture_mode" in
+  failure)
+    printf '%s\n' '{"type":"thread.started","thread_id":"fixture-thread"}'
+    printf '%s\n' '{"type":"turn.failed","error":"fixture failure"}'
+    ;;
+
+  hold)
+    printf '%s\n' '{"type":"thread.started","thread_id":"fixture-thread"}'
+    printf '%s\n' '{"type":"item.completed","item":{"type":"agent_message","text":"partial"}}'
+    sleep 2
+    printf '%s\n' '{"type":"turn.completed"}'
+    ;;
+
+  success)
+    printf '%s\n' '{"type":"thread.started","thread_id":"fixture-thread"}'
+    printf '%s\n' '{"type":"turn.started"}'
+    printf '%s\n' '{"type":"item.completed","item":{"type":"mcp_tool_call","id":"call-1","server":"fixture","tool":"read","arguments":{"path":"README.md"},"result":{"content":[]},"status":"completed"}}'
+    printf '%s\n' '{"type":"item.completed","item":{"type":"command_execution","id":"cmd-1","command":"pwd","aggregated_output":"/fixture","status":"completed"}}'
+    printf '%s\n' '{"type":"item.completed","item":{"type":"file_change","id":"file-1","changes":[],"status":"completed"}}'
+    printf '{"type":"item.completed","item":{"type":"agent_message","text":"fixture-%s"}}\n' "$mode"
+    printf '%s\n' '{"type":"turn.completed","usage":{"input_tokens":3,"output_tokens":2}}'
+    ;;
+
+  *)
+    printf 'unexpected fixture mode: %s\n' "$fixture_mode" >&2
+    exit 2
+    ;;
+esac

test/gen_agent/backends/codex_executable_conformance_test.exs

@@ -0,0 +1,183 @@
+defmodule GenAgent.Backends.CodexExecutableConformanceTest do
+  use ExUnit.Case, async: false
+
+  @moduletag capture_log: true
+
+  defmodule Agent do
+    use GenAgent
+
+    @impl true
+    def init_agent(opts) do
+      {observer, backend_opts} = Keyword.pop!(opts, :observer)
+      {:ok, backend_opts, %{observer: observer, responses: [], errors: []}}
+    end
+
+    @impl true
+    def handle_stream_event(event, state) do
+      send(state.observer, {:stream_event, event.kind, self()})
+      state
+    end
+
+    @impl true
+    def handle_response(ref, response, state) do
+      send(state.observer, {:completed, ref})
+      {:noreply, %{state | responses: [response | state.responses]}}
+    end
+
+    @impl true
+    def handle_error(ref, reason, state) do
+      send(state.observer, {:failed, ref, reason})
+      {:noreply, %{state | errors: [reason | state.errors]}}
+    end
+  end
+
+  setup do
+    previous_runner = Application.get_env(:codex_wrapper, :runner)
+    Application.put_env(:codex_wrapper, :runner, CodexWrapper.Runner.Port)
+
+    directory =
+      Path.join(System.tmp_dir!(), "gen-agent-codex-cli-#{System.unique_integer([:positive])}")
+
+    File.mkdir_p!(directory)
+    binary = Path.join(directory, "codex-fixture")
+    File.cp!(Path.expand("../../fixtures/codex_cli.sh", __DIR__), binary)
+    File.chmod!(binary, 0o755)
+
+    on_exit(fn ->
+      if previous_runner do
+        Application.put_env(:codex_wrapper, :runner, previous_runner)
+      else
+        Application.delete_env(:codex_wrapper, :runner)
+      end
+
+      File.rm_rf!(directory)
+    end)
+
+    %{binary: binary, directory: directory}
+  end
+
+  defp start_agent(context, opts \\ []) do
+    name = "codex-executable-#{System.unique_integer([:positive])}"
+
+    agent_opts =
+      [
+        name: name,
+        backend: GenAgent.Backends.Codex,
+        observer: self(),
+        binary: context.binary,
+        working_dir: context.directory
+      ] ++ opts
+
+    assert {:ok, _pid} = GenAgent.start_agent(Agent, agent_opts)
+
+    on_exit(fn ->
+      if GenAgent.whereis(name), do: GenAgent.stop(name)
+    end)
+
+    name
+  end
+
+  test "real wrapper and Port runner preserve arguments, events, and native identity on resume",
+       context do
+    assert CodexWrapper.Runner.impl() == CodexWrapper.Runner.Port
+
+    name =
+      start_agent(context,
+        model: "fixture-model",
+        sandbox: :read_only,
+        approval_policy: :never,
+        config_overrides: ["mcp_servers.fixture.enabled=false"],
+        enabled_features: ["fixture_feature"],
+        disabled_features: ["other_feature"],
+        images: ["/fixture/image.png"]
+      )
+
+    assert {:ok, first} = GenAgent.ask(name, "first prompt")
+    assert first.text == "fixture-fresh"
+    assert first.session_id == "fixture-thread"
+    assert first.usage == %{input_tokens: 3, output_tokens: 2}
+
+    assert Enum.map(first.events, & &1.kind) ==
+             [
+               :tool_use,
+               :tool_result,
+               :tool_use,
+               :tool_result,
+               :tool_use,
+               :tool_result,
+               :text,
+               :usage,
+               :result
+             ]
+
+    assert Enum.map(Enum.take(first.events, 6), & &1.data["id"]) ==
+             ["call-1", "call-1", "cmd-1", "cmd-1", "file-1", "file-1"]
+
+    fresh_args = args(context.directory, :fresh)
+    assert hd(fresh_args) == "exec"
+    assert "--json" in fresh_args
+    assert "fixture-model" in fresh_args
+    assert "read-only" in fresh_args
+    assert "first prompt" == List.last(fresh_args)
+
+    assert {:ok, second} = GenAgent.ask(name, "follow-up prompt")
+    assert second.text == "fixture-resume"
+    assert second.session_id == "fixture-thread"
+
+    resume_args = args(context.directory, :resume)
+    assert Enum.take(resume_args, 2) == ["exec", "resume"]
+    assert "fixture-thread" in resume_args
+    assert "fixture-model" in resume_args
+    assert "fixture_feature" in resume_args
+    assert "other_feature" in resume_args
+    assert "/fixture/image.png" in resume_args
+    assert "follow-up prompt" == List.last(resume_args)
+    assert Enum.any?(resume_args, &String.contains?(&1, "approval_policy"))
+    assert Enum.any?(resume_args, &String.contains?(&1, "mcp_servers.fixture.enabled=false"))
+  end
+
+  test "terminal failure from the executable reaches GenAgent as an error", context do
+    name = start_agent(context)
+    assert {:error, "fixture failure"} = GenAgent.ask(name, "fail")
+    assert GenAgent.status(name).agent_state.errors == ["fixture failure"]
+    assert "fail" == List.last(args(context.directory, :fresh))
+  end
+
+  for action <- [:interrupt, :watchdog, :stop, :kill] do
+    @tag action: action
+    test "#{action} stops the BEAM task on the executable streaming path", context do
+      action = context.action
+      watchdog_ms = if action == :watchdog, do: 500, else: 5_000
+      name = start_agent(context, watchdog_ms: watchdog_ms)
+      assert {:ok, ref} = GenAgent.tell(name, "hold")
+      assert_receive {:stream_event, :text, task_pid}, 1_000
+      task_monitor = Process.monitor(task_pid)
+
+      case action do
+        :interrupt ->
+          assert :ok = GenAgent.interrupt(name)
+          assert_receive {:failed, ^ref, :interrupted}, 1_000
+          assert {:error, :interrupted} = GenAgent.poll(name, ref)
+
+        :watchdog ->
+          assert_receive {:failed, ^ref, :timeout}, 1_000
+          assert {:error, :timeout} = GenAgent.poll(name, ref)
+
+        :stop ->
+          assert :ok = GenAgent.stop(name)
+
+        :kill ->
+          Process.exit(GenAgent.whereis(name), :kill)
+      end
+
+      assert_receive {:DOWN, ^task_monitor, :process, ^task_pid, :killed}, 1_000
+    end
+  end
+
+  defp args(directory, mode) do
+    directory
+    |> Path.join("#{mode}.args")
+    |> File.read!()
+    |> String.split("\n", trim: true)
+  end
+end