diff --git a/mix.lock b/mix.lock index 28a13c1..2163cd9 100644 --- a/mix.lock +++ b/mix.lock @@ -7,7 +7,7 @@ "erlex": {:hex, :erlex, "0.2.9", "7debbbaa9f4f368b8cd648983e0f1d7963028508e9c59e9d4ed504e94ef52a55", [:mix], [], "hexpm", "8cfffc0ec7159e6d73de2ab28a588064de80f88b2798d5cbe4482cbbc200178b"}, "ex_doc": {:hex, :ex_doc, "0.40.4", "66f2e42bf588594d5a8aab31cad87f2ddad09d0da1b1a2f379340ec2c2e497cb", [:mix], [{:earmark_parser, "~> 1.4.46", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "6222b9e423d76584ee34df2c82a5ed72c2d53dc153f7f483ad28b378694186cc"}, "file_system": {:hex, :file_system, "1.1.1", "31864f4685b0148f25bd3fbef2b1228457c0c89024ad67f7a81a3ffbc0bbad3a", [:mix], [], "hexpm", "7a15ff97dfe526aeefb090a7a9d3d03aa907e100e262a0f8f7746b78f8f87a5d"}, - "gen_agent": {:hex, :gen_agent, "0.2.0", "22d16e4124a5913882a2a8855cfdf0d9c696ae0a13afc131cd0919613f8e76ec", [:mix], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9c2474e32f81c81a9e1787dabcec93c78ca20520d7b9559184404e58b43f38e2"}, + "gen_agent": {:hex, :gen_agent, "0.2.1", "b55b3e24df8b9693763a0c34ade823b4bd92ed5c90e8ca64e5fb6e23e6eae973", [:mix], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "6355ca87301567f0f6aee98bef61de797ccd8cc0e1f93fbb81bece7ef8474dde"}, "jason": {:hex, :jason, "1.4.5", "2e3a008590b0b8d7388c20293e9dcc9cf3e5d642fd2a114e4cbbb52e595d940a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "b0c823996102bcd0239b3c2444eb00409b72f6a140c1950bc8b457d836b30684"}, "makeup": {:hex, :makeup, "1.2.2", "882d46dc0905e9ff7abf2aab61a7e6b3dcc555533977d8a23b06019e6c89ac94", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "9a1a24e5b343b8ae16abea0822c10a6f75da27af7fa802ada5251f7579bfccfa"}, "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"}, diff --git a/test/fixtures/codex_cli.sh b/test/fixtures/codex_cli.sh new file mode 100644 index 0000000..6f8510c --- /dev/null +++ b/test/fixtures/codex_cli.sh @@ -0,0 +1,52 @@ +#!/bin/sh +set -eu + +fixture_dir=$(cd "$(dirname "$0")" && pwd) + +mode=fresh +if [ "${2:-}" = "resume" ]; then + mode=resume +fi + +printf '%s\n' "$@" > "$fixture_dir/$mode.args" + +case "${*}" in + *fail*) + fixture_mode=failure + ;; + *hold*) + fixture_mode=hold + ;; + *) + fixture_mode=success + ;; +esac + +case "$fixture_mode" in + failure) + printf '%s\n' '{"type":"thread.started","thread_id":"fixture-thread"}' + printf '%s\n' '{"type":"turn.failed","error":"fixture failure"}' + ;; + + hold) + printf '%s\n' '{"type":"thread.started","thread_id":"fixture-thread"}' + printf '%s\n' '{"type":"item.completed","item":{"type":"agent_message","text":"partial"}}' + sleep 2 + printf '%s\n' '{"type":"turn.completed"}' + ;; + + success) + printf '%s\n' '{"type":"thread.started","thread_id":"fixture-thread"}' + printf '%s\n' '{"type":"turn.started"}' + printf '%s\n' '{"type":"item.completed","item":{"type":"mcp_tool_call","id":"call-1","server":"fixture","tool":"read","arguments":{"path":"README.md"},"result":{"content":[]},"status":"completed"}}' + printf '%s\n' '{"type":"item.completed","item":{"type":"command_execution","id":"cmd-1","command":"pwd","aggregated_output":"/fixture","status":"completed"}}' + printf '%s\n' '{"type":"item.completed","item":{"type":"file_change","id":"file-1","changes":[],"status":"completed"}}' + printf '{"type":"item.completed","item":{"type":"agent_message","text":"fixture-%s"}}\n' "$mode" + printf '%s\n' '{"type":"turn.completed","usage":{"input_tokens":3,"output_tokens":2}}' + ;; + + *) + printf 'unexpected fixture mode: %s\n' "$fixture_mode" >&2 + exit 2 + ;; +esac diff --git a/test/gen_agent/backends/codex_executable_conformance_test.exs b/test/gen_agent/backends/codex_executable_conformance_test.exs new file mode 100644 index 0000000..e35c417 --- /dev/null +++ b/test/gen_agent/backends/codex_executable_conformance_test.exs @@ -0,0 +1,183 @@ +defmodule GenAgent.Backends.CodexExecutableConformanceTest do + use ExUnit.Case, async: false + + @moduletag capture_log: true + + defmodule Agent do + use GenAgent + + @impl true + def init_agent(opts) do + {observer, backend_opts} = Keyword.pop!(opts, :observer) + {:ok, backend_opts, %{observer: observer, responses: [], errors: []}} + end + + @impl true + def handle_stream_event(event, state) do + send(state.observer, {:stream_event, event.kind, self()}) + state + end + + @impl true + def handle_response(ref, response, state) do + send(state.observer, {:completed, ref}) + {:noreply, %{state | responses: [response | state.responses]}} + end + + @impl true + def handle_error(ref, reason, state) do + send(state.observer, {:failed, ref, reason}) + {:noreply, %{state | errors: [reason | state.errors]}} + end + end + + setup do + previous_runner = Application.get_env(:codex_wrapper, :runner) + Application.put_env(:codex_wrapper, :runner, CodexWrapper.Runner.Port) + + directory = + Path.join(System.tmp_dir!(), "gen-agent-codex-cli-#{System.unique_integer([:positive])}") + + File.mkdir_p!(directory) + binary = Path.join(directory, "codex-fixture") + File.cp!(Path.expand("../../fixtures/codex_cli.sh", __DIR__), binary) + File.chmod!(binary, 0o755) + + on_exit(fn -> + if previous_runner do + Application.put_env(:codex_wrapper, :runner, previous_runner) + else + Application.delete_env(:codex_wrapper, :runner) + end + + File.rm_rf!(directory) + end) + + %{binary: binary, directory: directory} + end + + defp start_agent(context, opts \\ []) do + name = "codex-executable-#{System.unique_integer([:positive])}" + + agent_opts = + [ + name: name, + backend: GenAgent.Backends.Codex, + observer: self(), + binary: context.binary, + working_dir: context.directory + ] ++ opts + + assert {:ok, _pid} = GenAgent.start_agent(Agent, agent_opts) + + on_exit(fn -> + if GenAgent.whereis(name), do: GenAgent.stop(name) + end) + + name + end + + test "real wrapper and Port runner preserve arguments, events, and native identity on resume", + context do + assert CodexWrapper.Runner.impl() == CodexWrapper.Runner.Port + + name = + start_agent(context, + model: "fixture-model", + sandbox: :read_only, + approval_policy: :never, + config_overrides: ["mcp_servers.fixture.enabled=false"], + enabled_features: ["fixture_feature"], + disabled_features: ["other_feature"], + images: ["/fixture/image.png"] + ) + + assert {:ok, first} = GenAgent.ask(name, "first prompt") + assert first.text == "fixture-fresh" + assert first.session_id == "fixture-thread" + assert first.usage == %{input_tokens: 3, output_tokens: 2} + + assert Enum.map(first.events, & &1.kind) == + [ + :tool_use, + :tool_result, + :tool_use, + :tool_result, + :tool_use, + :tool_result, + :text, + :usage, + :result + ] + + assert Enum.map(Enum.take(first.events, 6), & &1.data["id"]) == + ["call-1", "call-1", "cmd-1", "cmd-1", "file-1", "file-1"] + + fresh_args = args(context.directory, :fresh) + assert hd(fresh_args) == "exec" + assert "--json" in fresh_args + assert "fixture-model" in fresh_args + assert "read-only" in fresh_args + assert "first prompt" == List.last(fresh_args) + + assert {:ok, second} = GenAgent.ask(name, "follow-up prompt") + assert second.text == "fixture-resume" + assert second.session_id == "fixture-thread" + + resume_args = args(context.directory, :resume) + assert Enum.take(resume_args, 2) == ["exec", "resume"] + assert "fixture-thread" in resume_args + assert "fixture-model" in resume_args + assert "fixture_feature" in resume_args + assert "other_feature" in resume_args + assert "/fixture/image.png" in resume_args + assert "follow-up prompt" == List.last(resume_args) + assert Enum.any?(resume_args, &String.contains?(&1, "approval_policy")) + assert Enum.any?(resume_args, &String.contains?(&1, "mcp_servers.fixture.enabled=false")) + end + + test "terminal failure from the executable reaches GenAgent as an error", context do + name = start_agent(context) + assert {:error, "fixture failure"} = GenAgent.ask(name, "fail") + assert GenAgent.status(name).agent_state.errors == ["fixture failure"] + assert "fail" == List.last(args(context.directory, :fresh)) + end + + for action <- [:interrupt, :watchdog, :stop, :kill] do + @tag action: action + test "#{action} stops the BEAM task on the executable streaming path", context do + action = context.action + watchdog_ms = if action == :watchdog, do: 500, else: 5_000 + name = start_agent(context, watchdog_ms: watchdog_ms) + assert {:ok, ref} = GenAgent.tell(name, "hold") + assert_receive {:stream_event, :text, task_pid}, 1_000 + task_monitor = Process.monitor(task_pid) + + case action do + :interrupt -> + assert :ok = GenAgent.interrupt(name) + assert_receive {:failed, ^ref, :interrupted}, 1_000 + assert {:error, :interrupted} = GenAgent.poll(name, ref) + + :watchdog -> + assert_receive {:failed, ^ref, :timeout}, 1_000 + assert {:error, :timeout} = GenAgent.poll(name, ref) + + :stop -> + assert :ok = GenAgent.stop(name) + + :kill -> + Process.exit(GenAgent.whereis(name), :kill) + end + + assert_receive {:DOWN, ^task_monitor, :process, ^task_pid, :killed}, 1_000 + end + end + + defp args(directory, mode) do + directory + |> Path.join("#{mode}.args") + |> File.read!() + |> String.split("\n", trim: true) + end +end