diff --git a/changes/3329-tailnet-watch-wrapper.md b/changes/3329-tailnet-watch-wrapper.md new file mode 100644 index 0000000000..44e26d2741 --- /dev/null +++ b/changes/3329-tailnet-watch-wrapper.md @@ -0,0 +1,11 @@ +--- +section: Fixed +--- + +## en + +- The Host no longer hangs on shutdown or plugin reload when `tailscale` on PATH is a shell wrapper that starts the real binary as a child (as some macOS installs do): the tailnet watcher now runs in its own process group, stopping it also stops the real Tailscale process, and disposal no longer waits on a pipe a leftover process holds open ([#3329](https://github.com/BotHarness/DeepSeekBot/pull/3329)). + +## zh + +- 当 PATH 上的 `tailscale` 是一个以子进程方式启动真实程序的 shell 包装脚本时(部分 macOS 安装如此),Host 关闭或插件重载不再卡住:tailnet 监听进程改为在独立进程组中运行,停止时会一并结束真实的 Tailscale 进程,释放时也不再等待残留进程占用的管道([#3329](https://github.com/BotHarness/DeepSeekBot/pull/3329))。 diff --git a/packages/core/src/devices/bot-tailnet-watch.ts b/packages/core/src/devices/bot-tailnet-watch.ts index cef186bb2c..699032c312 100644 --- a/packages/core/src/devices/bot-tailnet-watch.ts +++ b/packages/core/src/devices/bot-tailnet-watch.ts @@ -74,15 +74,39 @@ export function createTailnetNotificationReader(changed: (change: TailnetChange) }; } +export function spawnOwnedProcess(file: string, args: readonly string[]): ChildProcess { + return spawn(file, [...args], { + stdio: ['ignore', 'pipe', 'ignore'], + windowsHide: true, + detached: process.platform !== 'win32', + }); +} + +export function stopOwnedProcess(child: ChildProcess): void { + if (child.pid === undefined) return; + if (process.platform !== 'win32') { + try { + process.kill(-child.pid, 'SIGTERM'); + return; + } catch { + child.kill(); + return; + } + } + child.kill(); +} + export function watchTailnet( changed: (change: TailnetChange) => void, stopped: (code: 'tailnet-watch-unavailable' | 'tailnet-watch-invalid') => void, startProcess: (binary: string) => ChildProcess = (binary) => - spawn( - binary, - ['debug', 'watch-ipn', '--initial-status', '--peer-changes', '--peer-patches=false'], - { stdio: ['ignore', 'pipe', 'ignore'], windowsHide: true }, - ), + spawnOwnedProcess(binary, [ + 'debug', + 'watch-ipn', + '--initial-status', + '--peer-changes', + '--peer-patches=false', + ]), ): () => Promise { const binaries = process.platform === 'darwin' @@ -94,12 +118,13 @@ export function watchTailnet( const fail = (code: 'tailnet-watch-unavailable' | 'tailnet-watch-invalid') => { if (closed) return; closed = true; - if (child?.pid !== undefined) child.kill(); + if (child !== undefined) stopOwnedProcess(child); void Promise.all(exits).then(() => stopped(code)); }; const launch = (index: number) => { const process = startProcess(binaries[index]!); exits.push(new Promise((resolve) => process.once('close', () => resolve()))); + process.once('exit', () => process.stdout?.destroy()); child = process; const read = createTailnetNotificationReader(changed); process.stdout?.setEncoding('utf8'); @@ -123,7 +148,7 @@ export function watchTailnet( launch(0); return async () => { closed = true; - if (child?.pid !== undefined) child.kill(); + if (child !== undefined) stopOwnedProcess(child); await Promise.all(exits); }; } diff --git a/packages/core/test/bot-network-continuity.test.ts b/packages/core/test/bot-network-continuity.test.ts index 48727d046d..30b37e9866 100644 --- a/packages/core/test/bot-network-continuity.test.ts +++ b/packages/core/test/bot-network-continuity.test.ts @@ -11,6 +11,9 @@ import { receiveNetworkBotDm } from '../src/runtime/bot-network.js'; import { createDshBotAgentAdapter } from '../src/runtime/dsh-bot-agent-adapter.js'; import { FakeAgentHost } from './dsh-agent-host-fixture.js'; import { createTempRoot } from './helpers.js'; +import { loopbackAliasesAvailable } from './loopback-aliases.js'; + +const aliases = await loopbackAliasesAvailable(['127.0.0.31', '127.0.0.32']); async function dualRouteHosts(saved?: { homes: string[]; keys: Buffer[] }) { const nodes = [ @@ -246,220 +249,245 @@ async function explicitSender(core: ReturnType, rootId = 'roo ); } -it('keeps one DM when a new explicit send follows loss of the previously selected route', async () => { - const f = await dualRouteHosts(); - try { - const left = f.cores[0]!; - const contacts = await left.runtime.runDirectBotTool!( - 'bot', - 'list_bot_contacts', - {}, - new AbortController().signal, - ); - expect(contacts.result).toMatchObject({ contacts: [{ botId: 'remote:right:bot' }] }); - const send = await explicitSender(left); - await expect(send('FIRST')).resolves.toBeDefined(); - const channel = left.channels.list().find((item) => item.remoteBot)!; - expect(f.delivered).toEqual(['tailnet:FIRST']); - f.setTailnetOnline(false); - await expect(send('SECOND')).resolves.toBeDefined(); - expect(f.delivered).toEqual(['tailnet:FIRST', 'connect:SECOND']); - expect( - left.channels - .list() - .filter((item) => item.remoteBot) - .map((item) => item.id), - ).toEqual([channel.id]); - expect(left.channels.readMessages(channel.id).map((message) => message.body)).toEqual([ - 'SECOND', - 'FIRST', - ]); - const remote = f.cores[1]!.channels.list().find((item) => item.remoteBot)!; - expect(f.cores[1]!.channels.readMessages(remote.id).map((message) => message.body)).toEqual([ - 'SECOND', - 'FIRST', - ]); - } finally { - await f.close(); - } -}, 120_000); - -it('does not resend a committed unknown delivery on the other live route, but accepts a later explicit message in the same DM', async () => { - const f = await dualRouteHosts(); - try { - const left = f.cores[0]!; - f.setTailnetOnline(false); - const send = await explicitSender(left); - await expect(send('FIRST')).resolves.toBeDefined(); - const channel = left.channels.list().find((item) => item.remoteBot)!; - expect(f.delivered).toEqual(['connect:FIRST']); - f.loseNextConnectAcknowledgement(); - await expect(send('UNKNOWN')).rejects.toMatchObject({ code: 'delivery-unknown' }); - expect(left.runtime.networkBotAvailability!('right', 'bot')).toBe('available'); - expect(f.delivered).toEqual(['connect:FIRST', 'connect:UNKNOWN']); - await expect(send('LATER')).resolves.toBeDefined(); - expect(f.delivered).toEqual(['connect:FIRST', 'connect:UNKNOWN', 'tailnet:LATER']); - expect( - left.channels - .list() - .filter((item) => item.remoteBot) - .map((item) => item.id), - ).toEqual([channel.id]); - expect(left.channels.readMessages(channel.id).map((message) => message.body)).toEqual([ - 'LATER', - 'FIRST', - ]); - const remote = f.cores[1]!.channels.list().find((item) => item.remoteBot)!; - expect(f.cores[1]!.channels.readMessages(remote.id).map((message) => message.body)).toEqual([ - 'LATER', - 'UNKNOWN', - 'FIRST', - ]); - } finally { - await f.close(); - } -}, 120_000); - -it('retains one DM across both-route failure and Host restart without delivering the failed message later', async () => { - let f = await dualRouteHosts(); - try { - const send = await explicitSender(f.cores[0]!); - await expect(send('FIRST')).resolves.toBeDefined(); - const localId = f.cores[0]!.channels.list().find((item) => item.remoteBot)!.id; - const remoteId = f.cores[1]!.channels.list().find((item) => item.remoteBot)!.id; - f.setTailnetOnline(false); - f.setConnectOnline(false); - await expect(send('OFFLINE')).rejects.toMatchObject({ code: 'unable-to-deliver' }); - expect(f.delivered).toEqual(['tailnet:FIRST']); - expect(f.cores[0]!.runtime.networkBotAvailability!('right', 'bot')).toBe('unavailable'); - f.setConnectOnline(true); - await expect(send('RECOVERED')).resolves.toBeDefined(); - expect(f.delivered).toEqual(['tailnet:FIRST', 'connect:RECOVERED']); - const saved = f.saved; - await f.close(); - f = await dualRouteHosts(saved); - await Promise.all(f.cores.map((core) => core.runtime.whenIdle())); - expect(f.delivered).toEqual([]); - expect(f.cores[0]!.channels.readMessages(localId).map((message) => message.body)).toEqual([ - 'RECOVERED', - 'FIRST', - ]); - expect(f.cores[1]!.channels.readMessages(remoteId).map((message) => message.body)).toEqual([ - 'RECOVERED', - 'FIRST', - ]); - const next = await explicitSender(f.cores[0]!, 'new-request'); - await expect(next('LATER')).resolves.toBeDefined(); - expect(f.delivered).toEqual(['tailnet:LATER']); - for (const [index, id] of [localId, remoteId].entries()) { +it.skipIf(!aliases)( + 'keeps one DM when a new explicit send follows loss of the previously selected route', + async () => { + const f = await dualRouteHosts(); + try { + const left = f.cores[0]!; + const contacts = await left.runtime.runDirectBotTool!( + 'bot', + 'list_bot_contacts', + {}, + new AbortController().signal, + ); + expect(contacts.result).toMatchObject({ contacts: [{ botId: 'remote:right:bot' }] }); + const send = await explicitSender(left); + await expect(send('FIRST')).resolves.toBeDefined(); + const channel = left.channels.list().find((item) => item.remoteBot)!; + expect(f.delivered).toEqual(['tailnet:FIRST']); + f.setTailnetOnline(false); + await expect(send('SECOND')).resolves.toBeDefined(); + expect(f.delivered).toEqual(['tailnet:FIRST', 'connect:SECOND']); expect( - f.cores[index]!.channels.list() + left.channels + .list() .filter((item) => item.remoteBot) .map((item) => item.id), - ).toEqual([id]); - expect(f.cores[index]!.channels.readMessages(id).map((message) => message.body)).toEqual([ - 'LATER', - 'RECOVERED', + ).toEqual([channel.id]); + expect(left.channels.readMessages(channel.id).map((message) => message.body)).toEqual([ + 'SECOND', 'FIRST', ]); + const remote = f.cores[1]!.channels.list().find((item) => item.remoteBot)!; + expect(f.cores[1]!.channels.readMessages(remote.id).map((message) => message.body)).toEqual([ + 'SECOND', + 'FIRST', + ]); + } finally { + await f.close(); } - } finally { - await f.close(); - } -}, 120_000); - -it('sends through the remaining common scope but refuses after both memberships withdraw', async () => { - const f = await dualRouteHosts(); - try { - const left = f.cores[0]!; - const send = await explicitSender(left); - await expect(send('FIRST')).resolves.toBeDefined(); - const channel = left.channels.list().find((item) => item.remoteBot)!; - f.withdrawTailnet(); - await expect(send('REMAINING_SCOPE')).resolves.toBeDefined(); - expect(f.delivered).toEqual(['tailnet:FIRST', 'connect:REMAINING_SCOPE']); - expect(left.runtime.networkBotAvailability!('right', 'bot')).toBe('available'); - f.withdrawConnect(); - await expect(send('NO_SCOPE')).rejects.toMatchObject({ - code: expect.stringMatching(/^(network-denied|unable-to-deliver)$/u), - }); - const contacts = await left.runtime.runDirectBotTool!( - 'bot', - 'list_bot_contacts', - {}, - new AbortController().signal, - ); - expect(contacts.result).toMatchObject({ contacts: [] }); - expect(f.delivered).toEqual(['tailnet:FIRST', 'connect:REMAINING_SCOPE']); - expect(left.channels.readMessages(channel.id).map((message) => message.body)).toEqual([ - 'REMAINING_SCOPE', - 'FIRST', - ]); - } finally { - await f.close(); - } -}, 120_000); + }, + 120_000, +); -it('keeps updated profile facts and a reply on the same identity across routes despite a same-name colleague', async () => { - const f = await dualRouteHosts(); - try { - const [left, right] = f.cores; - right!.registry.create({ slug: 'twin', displayName: 'Developer' }); - right!.registry.update('bot', { description: 'Initial responsibility' }); - const query = () => - left!.runtime.runDirectBotTool!('bot', 'list_bot_contacts', {}, new AbortController().signal); - expect((await query()).result).toMatchObject({ - contacts: [ - { botId: 'remote:right:bot', displayName: 'Developer' }, - { botId: 'remote:right:twin', displayName: 'Developer' }, - ], - }); - const send = await explicitSender(left!); - await send('FIRST'); - const localId = left!.channels.list().find((item) => item.remoteBot)!.id; - const remoteId = right!.channels.list().find((item) => item.remoteBot)!.id; - right!.registry.update('bot', { description: 'iOS development and debugging' }); - f.setDeviceModel(1, 'Test workstation'); - f.setTailnetOnline(false); - await send('SECOND'); - expect((await query()).result).toMatchObject({ - contacts: [ - { - botId: 'remote:right:bot', - description: 'iOS development and debugging', - device: { model: 'Test workstation' }, - }, - { botId: 'remote:right:twin', displayName: 'Developer' }, - ], - }); - const accepted = right!.inboxHistory.list('bot', { - channelId: remoteId, - query: 'SECOND', - }).items; - expect(accepted).toHaveLength(1); - await right!.runtime.runDirectBotTool!( - 'bot', - 'bot_send_message', - { bot_id: 'remote:left:bot', body: 'REPLY' }, - new AbortController().signal, - undefined, - accepted[0]!.sourceEventId, - ); - expect(f.delivered).toEqual(['tailnet:FIRST', 'connect:SECOND', 'connect:REPLY']); - for (const [index, id] of [localId, remoteId].entries()) { +it.skipIf(!aliases)( + 'does not resend a committed unknown delivery on the other live route, but accepts a later explicit message in the same DM', + async () => { + const f = await dualRouteHosts(); + try { + const left = f.cores[0]!; + f.setTailnetOnline(false); + const send = await explicitSender(left); + await expect(send('FIRST')).resolves.toBeDefined(); + const channel = left.channels.list().find((item) => item.remoteBot)!; + expect(f.delivered).toEqual(['connect:FIRST']); + f.loseNextConnectAcknowledgement(); + await expect(send('UNKNOWN')).rejects.toMatchObject({ code: 'delivery-unknown' }); + expect(left.runtime.networkBotAvailability!('right', 'bot')).toBe('available'); + expect(f.delivered).toEqual(['connect:FIRST', 'connect:UNKNOWN']); + await expect(send('LATER')).resolves.toBeDefined(); + expect(f.delivered).toEqual(['connect:FIRST', 'connect:UNKNOWN', 'tailnet:LATER']); expect( - f.cores[index]!.channels.list() + left.channels + .list() .filter((item) => item.remoteBot) .map((item) => item.id), - ).toEqual([id]); - expect(f.cores[index]!.channels.readMessages(id).map((message) => message.body)).toEqual([ - 'REPLY', - 'SECOND', + ).toEqual([channel.id]); + expect(left.channels.readMessages(channel.id).map((message) => message.body)).toEqual([ + 'LATER', + 'FIRST', + ]); + const remote = f.cores[1]!.channels.list().find((item) => item.remoteBot)!; + expect(f.cores[1]!.channels.readMessages(remote.id).map((message) => message.body)).toEqual([ + 'LATER', + 'UNKNOWN', 'FIRST', ]); + } finally { + await f.close(); } - expect(right!.inboxHistory.list('twin').items).toEqual([]); - } finally { - await f.close(); - } -}, 120_000); + }, + 120_000, +); + +it.skipIf(!aliases)( + 'retains one DM across both-route failure and Host restart without delivering the failed message later', + async () => { + let f = await dualRouteHosts(); + try { + const send = await explicitSender(f.cores[0]!); + await expect(send('FIRST')).resolves.toBeDefined(); + const localId = f.cores[0]!.channels.list().find((item) => item.remoteBot)!.id; + const remoteId = f.cores[1]!.channels.list().find((item) => item.remoteBot)!.id; + f.setTailnetOnline(false); + f.setConnectOnline(false); + await expect(send('OFFLINE')).rejects.toMatchObject({ code: 'unable-to-deliver' }); + expect(f.delivered).toEqual(['tailnet:FIRST']); + expect(f.cores[0]!.runtime.networkBotAvailability!('right', 'bot')).toBe('unavailable'); + f.setConnectOnline(true); + await expect(send('RECOVERED')).resolves.toBeDefined(); + expect(f.delivered).toEqual(['tailnet:FIRST', 'connect:RECOVERED']); + const saved = f.saved; + await f.close(); + f = await dualRouteHosts(saved); + await Promise.all(f.cores.map((core) => core.runtime.whenIdle())); + expect(f.delivered).toEqual([]); + expect(f.cores[0]!.channels.readMessages(localId).map((message) => message.body)).toEqual([ + 'RECOVERED', + 'FIRST', + ]); + expect(f.cores[1]!.channels.readMessages(remoteId).map((message) => message.body)).toEqual([ + 'RECOVERED', + 'FIRST', + ]); + const next = await explicitSender(f.cores[0]!, 'new-request'); + await expect(next('LATER')).resolves.toBeDefined(); + expect(f.delivered).toEqual(['tailnet:LATER']); + for (const [index, id] of [localId, remoteId].entries()) { + expect( + f.cores[index]!.channels.list() + .filter((item) => item.remoteBot) + .map((item) => item.id), + ).toEqual([id]); + expect(f.cores[index]!.channels.readMessages(id).map((message) => message.body)).toEqual([ + 'LATER', + 'RECOVERED', + 'FIRST', + ]); + } + } finally { + await f.close(); + } + }, + 120_000, +); + +it.skipIf(!aliases)( + 'sends through the remaining common scope but refuses after both memberships withdraw', + async () => { + const f = await dualRouteHosts(); + try { + const left = f.cores[0]!; + const send = await explicitSender(left); + await expect(send('FIRST')).resolves.toBeDefined(); + const channel = left.channels.list().find((item) => item.remoteBot)!; + f.withdrawTailnet(); + await expect(send('REMAINING_SCOPE')).resolves.toBeDefined(); + expect(f.delivered).toEqual(['tailnet:FIRST', 'connect:REMAINING_SCOPE']); + expect(left.runtime.networkBotAvailability!('right', 'bot')).toBe('available'); + f.withdrawConnect(); + await expect(send('NO_SCOPE')).rejects.toMatchObject({ + code: expect.stringMatching(/^(network-denied|unable-to-deliver)$/u), + }); + const contacts = await left.runtime.runDirectBotTool!( + 'bot', + 'list_bot_contacts', + {}, + new AbortController().signal, + ); + expect(contacts.result).toMatchObject({ contacts: [] }); + expect(f.delivered).toEqual(['tailnet:FIRST', 'connect:REMAINING_SCOPE']); + expect(left.channels.readMessages(channel.id).map((message) => message.body)).toEqual([ + 'REMAINING_SCOPE', + 'FIRST', + ]); + } finally { + await f.close(); + } + }, + 120_000, +); + +it.skipIf(!aliases)( + 'keeps updated profile facts and a reply on the same identity across routes despite a same-name colleague', + async () => { + const f = await dualRouteHosts(); + try { + const [left, right] = f.cores; + right!.registry.create({ slug: 'twin', displayName: 'Developer' }); + right!.registry.update('bot', { description: 'Initial responsibility' }); + const query = () => + left!.runtime.runDirectBotTool!( + 'bot', + 'list_bot_contacts', + {}, + new AbortController().signal, + ); + expect((await query()).result).toMatchObject({ + contacts: [ + { botId: 'remote:right:bot', displayName: 'Developer' }, + { botId: 'remote:right:twin', displayName: 'Developer' }, + ], + }); + const send = await explicitSender(left!); + await send('FIRST'); + const localId = left!.channels.list().find((item) => item.remoteBot)!.id; + const remoteId = right!.channels.list().find((item) => item.remoteBot)!.id; + right!.registry.update('bot', { description: 'iOS development and debugging' }); + f.setDeviceModel(1, 'Test workstation'); + f.setTailnetOnline(false); + await send('SECOND'); + expect((await query()).result).toMatchObject({ + contacts: [ + { + botId: 'remote:right:bot', + description: 'iOS development and debugging', + device: { model: 'Test workstation' }, + }, + { botId: 'remote:right:twin', displayName: 'Developer' }, + ], + }); + const accepted = right!.inboxHistory.list('bot', { + channelId: remoteId, + query: 'SECOND', + }).items; + expect(accepted).toHaveLength(1); + await right!.runtime.runDirectBotTool!( + 'bot', + 'bot_send_message', + { bot_id: 'remote:left:bot', body: 'REPLY' }, + new AbortController().signal, + undefined, + accepted[0]!.sourceEventId, + ); + expect(f.delivered).toEqual(['tailnet:FIRST', 'connect:SECOND', 'connect:REPLY']); + for (const [index, id] of [localId, remoteId].entries()) { + expect( + f.cores[index]!.channels.list() + .filter((item) => item.remoteBot) + .map((item) => item.id), + ).toEqual([id]); + expect(f.cores[index]!.channels.readMessages(id).map((message) => message.body)).toEqual([ + 'REPLY', + 'SECOND', + 'FIRST', + ]); + } + expect(right!.inboxHistory.list('twin').items).toEqual([]); + } finally { + await f.close(); + } + }, + 120_000, +); diff --git a/packages/core/test/bot-tailnet-bridge.test.ts b/packages/core/test/bot-tailnet-bridge.test.ts index 39092ad050..c4fdd07f23 100644 --- a/packages/core/test/bot-tailnet-bridge.test.ts +++ b/packages/core/test/bot-tailnet-bridge.test.ts @@ -10,8 +10,14 @@ import { expect, it } from 'vitest'; import { createTailnetBotBridge } from '../src/devices/bot-tailnet-bridge.js'; import type { BotTailnet } from '../src/devices/bot-tailnet.js'; import type { TailnetChange } from '../src/devices/bot-tailnet-watch.js'; +import { loopbackAliasesAvailable } from './loopback-aliases.js'; -it.each(['background', 'explicit'])( +const directoryAliases = await loopbackAliasesAvailable(['127.0.0.10', '127.0.0.11']); +const withdrawalAliases = await loopbackAliasesAvailable(['127.0.0.8', '127.0.0.9']); +const rebindAliases = await loopbackAliasesAvailable(['127.0.0.2', '127.0.0.3', '127.0.0.4']); +const relationshipAliases = await loopbackAliasesAvailable(['127.0.0.2', '127.0.0.3']); + +it.skipIf(!directoryAliases).each(['background', 'explicit'])( 'cancels an older %s directory read when a fresh query starts', async (kind) => { const self = { nodeId: 'self', address: '127.0.0.10' }; @@ -65,7 +71,7 @@ it.each(['background', 'explicit'])( }, ); -it.each([false, true])( +it.skipIf(!withdrawalAliases).each([false, true])( 'waits for observed withdrawal before trusting a returned peer (cancellation fails: %s)', async (fails) => { const self = { nodeId: 'self', address: '127.0.0.8' }; @@ -142,54 +148,57 @@ it.each([false, true])( }, ); -it('rebinds its own listener after the local tailnet identity changes', async () => { - let self = { nodeId: 'initial-node', address: '127.0.0.2' }; - const bridge = createTailnetBotBridge({ - hostId: 'local', - key: async () => randomBytes(32), - directory: () => [], - async receive() {}, - ports: [43273], - tailnet: { - async snapshot() { - return { self, peers: [] }; - }, - async admit() { - return { nodeId: 'caller', address: '127.0.0.3' }; - }, - }, - }); - const query = (address: string) => - new Promise((resolve) => { - const call = request( - { - hostname: address, - port: 43273, - path: '/botharness-bots/v1/contacts?offset=0', - localAddress: '127.0.0.3', - agent: false, - timeout: 1000, +it.skipIf(!rebindAliases)( + 'rebinds its own listener after the local tailnet identity changes', + async () => { + let self = { nodeId: 'initial-node', address: '127.0.0.2' }; + const bridge = createTailnetBotBridge({ + hostId: 'local', + key: async () => randomBytes(32), + directory: () => [], + async receive() {}, + ports: [43273], + tailnet: { + async snapshot() { + return { self, peers: [] }; }, - (response) => { - response.resume(); - resolve(response.statusCode!); + async admit() { + return { nodeId: 'caller', address: '127.0.0.3' }; }, - ); - call.on('error', () => resolve('offline')); - call.on('timeout', () => call.destroy()); - call.end(); + }, }); - try { - await bridge.start(); - expect(await query('127.0.0.2')).toBe(200); - self = { nodeId: 'replacement-node', address: '127.0.0.4' }; - await bridge.refresh?.(); - expect(await query('127.0.0.4')).toBe(200); - expect(await query('127.0.0.2')).toBe('offline'); - } finally { - await bridge.close(); - } -}); + const query = (address: string) => + new Promise((resolve) => { + const call = request( + { + hostname: address, + port: 43273, + path: '/botharness-bots/v1/contacts?offset=0', + localAddress: '127.0.0.3', + agent: false, + timeout: 1000, + }, + (response) => { + response.resume(); + resolve(response.statusCode!); + }, + ); + call.on('error', () => resolve('offline')); + call.on('timeout', () => call.destroy()); + call.end(); + }); + try { + await bridge.start(); + expect(await query('127.0.0.2')).toBe(200); + self = { nodeId: 'replacement-node', address: '127.0.0.4' }; + await bridge.refresh?.(); + expect(await query('127.0.0.4')).toBe(200); + expect(await query('127.0.0.2')).toBe('offline'); + } finally { + await bridge.close(); + } + }, +); it('checks persisted membership independently of online routes after a bridge restart', async () => { const self = { nodeId: 'local-node', address: '127.0.0.2' }; @@ -244,7 +253,7 @@ it('checks persisted membership independently of online routes after a bridge re } }); -it.each(['changed', 'legacy'])( +it.skipIf(!relationshipAliases).each(['changed', 'legacy'])( 'discovers and sends exactly once while replacing a %s relationship', async (relationship) => { const keys = [randomBytes(32), randomBytes(32)]; diff --git a/packages/core/test/bot-tailnet-watch.test.ts b/packages/core/test/bot-tailnet-watch.test.ts index 92e7730f4d..b24ee8cebe 100644 --- a/packages/core/test/bot-tailnet-watch.test.ts +++ b/packages/core/test/bot-tailnet-watch.test.ts @@ -1,6 +1,29 @@ import { expect, it, vi } from 'vitest'; -import { spawn } from 'node:child_process'; -import { createTailnetNotificationReader, watchTailnet } from '../src/devices/bot-tailnet-watch.js'; +import { execFileSync, spawn } from 'node:child_process'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { + createTailnetNotificationReader, + spawnOwnedProcess, + watchTailnet, +} from '../src/devices/bot-tailnet-watch.js'; +import { createTempRoot } from './helpers.js'; + +function alive(pid: number): boolean { + try { + process.kill(pid, 0); + } catch { + return false; + } + try { + const state = execFileSync('ps', ['-o', 'stat=', '-p', String(pid)], { + encoding: 'utf8', + }).trim(); + return state !== '' && !state.startsWith('Z'); + } catch { + return false; + } +} it.each(['immediate', 'after-error'] as const)( 'never signals a subprocess that failed to start when disposed %s', @@ -58,6 +81,35 @@ it('does not finish disposal until its owned subprocess has exited', async () => } }); +it.skipIf(process.platform === 'win32')( + 'stops a wrapper-launched watcher and the process it started when disposed', + async () => { + const pidFile = join(createTempRoot('botharness-tailnet-wrapper-'), 'inner.pid'); + const inner = `require('node:fs').writeFileSync(${JSON.stringify(pidFile)}, String(process.pid)); setInterval(() => {}, 1000);`; + const wrapper = `"${process.execPath}" -e "${inner.replaceAll('"', '\\"')}"\nexit $?`; + const stop = watchTailnet( + () => {}, + () => {}, + () => spawnOwnedProcess('/bin/sh', ['-c', wrapper]), + ); + let innerPid: number | undefined; + try { + await vi.waitFor( + () => { + innerPid = Number(readFileSync(pidFile, 'utf8')); + expect(innerPid).toBeGreaterThan(0); + }, + { timeout: 5000 }, + ); + await stop(); + await vi.waitFor(() => expect(alive(innerPid!)).toBe(false), { timeout: 5000 }); + } finally { + if (innerPid !== undefined && alive(innerPid)) process.kill(innerPid, 'SIGKILL'); + } + }, + 10_000, +); + it('retains a removal followed by re-addition in the same native stream chunk', () => { const changes: unknown[] = []; const reader = createTailnetNotificationReader((change) => changes.push(change)); diff --git a/packages/core/test/loopback-aliases.ts b/packages/core/test/loopback-aliases.ts new file mode 100644 index 0000000000..f10925f2ef --- /dev/null +++ b/packages/core/test/loopback-aliases.ts @@ -0,0 +1,27 @@ +import { createServer } from 'node:net'; + +async function canListen(address: string): Promise { + const server = createServer(); + try { + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, address, () => resolve()); + }); + return true; + } catch { + return false; + } finally { + if (server.listening) await new Promise((resolve) => server.close(() => resolve())); + } +} + +export async function loopbackAliasesAvailable(addresses: readonly string[]): Promise { + const results = await Promise.all(addresses.map(canListen)); + const missing = addresses.filter((_, index) => !results[index]); + if (missing.length > 0) + console.warn( + `Skipping real-bridge tailnet tests: cannot bind ${missing.join(', ')}. ` + + 'On macOS add them with `sudo ifconfig lo0 alias
` to run these tests.', + ); + return missing.length === 0; +} diff --git a/packages/core/test/network-lifecycle.test.ts b/packages/core/test/network-lifecycle.test.ts index eff783ac06..dec477647d 100644 --- a/packages/core/test/network-lifecycle.test.ts +++ b/packages/core/test/network-lifecycle.test.ts @@ -7,218 +7,228 @@ import { receiveNetworkBotDm } from '../src/runtime/bot-network.js'; import { createDshBotAgentAdapter } from '../src/runtime/dsh-bot-agent-adapter.js'; import { FakeAgentHost } from './dsh-agent-host-fixture.js'; import { createTempRoot } from './helpers.js'; +import { loopbackAliasesAvailable } from './loopback-aliases.js'; -it('uses public tools over real bridges for replacement colleagues and an explicitly chosen third Host', async () => { - const nodes = [ - { nodeId: 'a', address: '127.0.0.5' }, - { nodeId: 'b', address: '127.0.0.6' }, - { nodeId: 'c', address: '127.0.0.7' }, - ]; - const cores: Array> = []; - const bridges: Array> = []; - const adjacent = (index: number) => nodes.filter((_, other) => Math.abs(index - other) === 1); - try { - for (const [index, node] of nodes.entries()) { - const key = randomBytes(32); - const bridge = createTailnetBotBridge({ - hostId: node.nodeId, - key: async () => Uint8Array.from(key), - ports: [43275], - tailnet: { - async snapshot() { - return { self: node, peers: adjacent(index) }; +const aliases = await loopbackAliasesAvailable(['127.0.0.5', '127.0.0.6', '127.0.0.7']); + +it.skipIf(!aliases)( + 'uses public tools over real bridges for replacement colleagues and an explicitly chosen third Host', + async () => { + const nodes = [ + { nodeId: 'a', address: '127.0.0.5' }, + { nodeId: 'b', address: '127.0.0.6' }, + { nodeId: 'c', address: '127.0.0.7' }, + ]; + const cores: Array> = []; + const bridges: Array> = []; + const adjacent = (index: number) => nodes.filter((_, other) => Math.abs(index - other) === 1); + try { + for (const [index, node] of nodes.entries()) { + const key = randomBytes(32); + const bridge = createTailnetBotBridge({ + hostId: node.nodeId, + key: async () => Uint8Array.from(key), + ports: [43275], + tailnet: { + async snapshot() { + return { self: node, peers: adjacent(index) }; + }, + async admit(address) { + const peer = adjacent(index).find((candidate) => candidate.address === address); + if (!peer) throw new Error('tailnet-peer-denied'); + return peer; + }, }, - async admit(address) { - const peer = adjacent(index).find((candidate) => candidate.address === address); - if (!peer) throw new Error('tailnet-peer-denied'); - return peer; + knownKey: (host) => cores[index]!.channels.remoteHostKey!(host), + trustKey: (host, key, membership) => + cores[index]!.channels.trustRemoteHostKey!(host, key, membership), + knownMemberships: () => + cores[index]!.channels.list().flatMap((channel) => + channel.remoteBot?.membership + ? [{ hostId: channel.remoteBot.hostId, membership: channel.remoteBot.membership }] + : [], + ), + withdrawn: (host) => cores[index]!.runtime.withdrawNetworkMembership(host), + directory: () => + cores[index]!.registry.list() + .filter((bot) => !bot.paused) + .map((bot) => ({ + hostId: node.nodeId, + botSlug: bot.slug, + displayName: bot.displayName, + ...(bot.description === undefined ? {} : { description: bot.description }), + device: { model: null, platform: 'test' }, + capabilities: [], + })), + async receive(sender, message) { + const core = cores[index]!; + await receiveNetworkBotDm( + { + registry: core.registry, + channels: core.channels, + admitBotDmMessage: (channel, id) => core.runtime.admitBotDmMessage(channel, id), + }, + sender, + message, + ); }, - }, - knownKey: (host) => cores[index]!.channels.remoteHostKey!(host), - trustKey: (host, key, membership) => - cores[index]!.channels.trustRemoteHostKey!(host, key, membership), - knownMemberships: () => - cores[index]!.channels.list().flatMap((channel) => - channel.remoteBot?.membership - ? [{ hostId: channel.remoteBot.hostId, membership: channel.remoteBot.membership }] - : [], - ), - withdrawn: (host) => cores[index]!.runtime.withdrawNetworkMembership(host), - directory: () => - cores[index]!.registry.list() - .filter((bot) => !bot.paused) - .map((bot) => ({ - hostId: node.nodeId, - botSlug: bot.slug, - displayName: bot.displayName, - ...(bot.description === undefined ? {} : { description: bot.description }), - device: { model: null, platform: 'test' }, - capabilities: [], - })), - async receive(sender, message) { - const core = cores[index]!; - await receiveNetworkBotDm( - { - registry: core.registry, - channels: core.channels, - admitBotDmMessage: (channel, id) => core.runtime.admitBotDmMessage(channel, id), - }, - sender, - message, - ); - }, - }); - bridges.push(bridge); - cores.push( - createCore({ - dshHome: createTempRoot('network-lifecycle-'), - botNetwork: bridge, - agents: createDshBotAgentAdapter({ - agents: new FakeAgentHost({ kind: 'completed' }), - orchestratorCwd: () => '/memory', - ensureWorkspace: () => undefined, - defaultModel: { currentSelection: () => ({ provider: 'test', model: 'test' }) }, + }); + bridges.push(bridge); + cores.push( + createCore({ + dshHome: createTempRoot('network-lifecycle-'), + botNetwork: bridge, + agents: createDshBotAgentAdapter({ + agents: new FakeAgentHost({ kind: 'completed' }), + orchestratorCwd: () => '/memory', + ensureWorkspace: () => undefined, + defaultModel: { currentSelection: () => ({ provider: 'test', model: 'test' }) }, + }), }), - }), - ); - cores[index]!.registry.create({ slug: 'bot', displayName: 'Colleague' }); - await bridge.start(); - } - const [a, b, c] = cores; - const query = (index: number, slug = 'bot') => - cores[index]!.runtime.runDirectBotTool!( - slug, - 'list_bot_contacts', - {}, - new AbortController().signal, - ); - const send = (index: number, source: string, botId: string, body: string) => - cores[index]!.runtime.runDirectBotTool!( - 'bot', - 'bot_send_message', - { bot_id: botId, body }, - new AbortController().signal, - undefined, - source, - ); - const inbound = (index: number, body: string) => { - const core = cores[index]!; - const row = attachOperationalModule(core.operationalDatabase, 'test-network-lifecycle').read( - (db) => db.prepare('SELECT source_event_id FROM source_events WHERE body = ?').get(body), - ) as { source_event_id: string } | undefined; - expect(row).toBeDefined(); - return row!.source_event_id; - }; - const dm = a!.channels.getOrCreateDm('bot', 'Colleague')!; - await a!.channels.appendMessage(dm.id, { - id: 'root', - at: new Date().toISOString(), - author: { kind: 'human' }, - body: 'Start collaboration', - }); - const root = a!.channels.humanSendStatus!(dm.id, 'root')!.sourceEventId; - expect((await query(0)).result).toMatchObject({ contacts: [{ botId: 'remote:b:bot' }] }); - await send(0, root, 'remote:b:bot', 'Investigate'); - const originalChannel = a!.channels.list().find((channel) => channel.remoteBot)!; + ); + cores[index]!.registry.create({ slug: 'bot', displayName: 'Colleague' }); + await bridge.start(); + } + const [a, b, c] = cores; + const query = (index: number, slug = 'bot') => + cores[index]!.runtime.runDirectBotTool!( + slug, + 'list_bot_contacts', + {}, + new AbortController().signal, + ); + const send = (index: number, source: string, botId: string, body: string) => + cores[index]!.runtime.runDirectBotTool!( + 'bot', + 'bot_send_message', + { bot_id: botId, body }, + new AbortController().signal, + undefined, + source, + ); + const inbound = (index: number, body: string) => { + const core = cores[index]!; + const row = attachOperationalModule( + core.operationalDatabase, + 'test-network-lifecycle', + ).read((db) => + db.prepare('SELECT source_event_id FROM source_events WHERE body = ?').get(body), + ) as { source_event_id: string } | undefined; + expect(row).toBeDefined(); + return row!.source_event_id; + }; + const dm = a!.channels.getOrCreateDm('bot', 'Colleague')!; + await a!.channels.appendMessage(dm.id, { + id: 'root', + at: new Date().toISOString(), + author: { kind: 'human' }, + body: 'Start collaboration', + }); + const root = a!.channels.humanSendStatus!(dm.id, 'root')!.sourceEventId; + expect((await query(0)).result).toMatchObject({ contacts: [{ botId: 'remote:b:bot' }] }); + await send(0, root, 'remote:b:bot', 'Investigate'); + const originalChannel = a!.channels.list().find((channel) => channel.remoteBot)!; - b!.registry.setPaused('bot', true); - await expect(send(0, root, 'remote:b:bot', 'Stale archived contact')).rejects.toMatchObject({ - code: 'network-denied', - }); - expect((await query(0)).result).toMatchObject({ contacts: [] }); - b!.registry.setPaused('bot', false); - await query(0); - await b!.deletions.confirm('bot', b!.deletions.preview('bot').token, false); - await expect(send(0, root, 'remote:b:bot', 'Stale deleted contact')).rejects.toMatchObject({ - code: 'network-denied', - }); - b!.registry.create({ - slug: 'replacement', - displayName: 'Colleague', - description: 'New iOS specialist', - }); - expect((await query(0)).result).toMatchObject({ - contacts: [{ botId: 'remote:b:replacement', description: 'New iOS specialist' }], - }); - await send(0, root, 'remote:b:replacement', 'New identity'); - const replacement = a!.channels - .list() - .find((channel) => channel.remoteBot?.botSlug === 'replacement')!; - expect(replacement.id).not.toBe(originalChannel.id); - expect(a!.channels.readMessages(originalChannel.id).map((message) => message.body)).toEqual([ - 'Investigate', - ]); - expect(a!.channels.readMessages(replacement.id).map((message) => message.body)).toEqual([ - 'New identity', - ]); + b!.registry.setPaused('bot', true); + await expect(send(0, root, 'remote:b:bot', 'Stale archived contact')).rejects.toMatchObject({ + code: 'network-denied', + }); + expect((await query(0)).result).toMatchObject({ contacts: [] }); + b!.registry.setPaused('bot', false); + await query(0); + await b!.deletions.confirm('bot', b!.deletions.preview('bot').token, false); + await expect(send(0, root, 'remote:b:bot', 'Stale deleted contact')).rejects.toMatchObject({ + code: 'network-denied', + }); + b!.registry.create({ + slug: 'replacement', + displayName: 'Colleague', + description: 'New iOS specialist', + }); + expect((await query(0)).result).toMatchObject({ + contacts: [{ botId: 'remote:b:replacement', description: 'New iOS specialist' }], + }); + await send(0, root, 'remote:b:replacement', 'New identity'); + const replacement = a!.channels + .list() + .find((channel) => channel.remoteBot?.botSlug === 'replacement')!; + expect(replacement.id).not.toBe(originalChannel.id); + expect(a!.channels.readMessages(originalChannel.id).map((message) => message.body)).toEqual([ + 'Investigate', + ]); + expect(a!.channels.readMessages(replacement.id).map((message) => message.body)).toEqual([ + 'New identity', + ]); - expect((await query(1, 'replacement')).result).toMatchObject({ - contacts: expect.arrayContaining([expect.objectContaining({ botId: 'remote:c:bot' })]), - }); - const secondSource = inbound(1, 'New identity'); - await b!.runtime.runDirectBotTool!( - 'replacement', - 'bot_send_message', - { - bot_id: 'remote:c:bot', - body: 'Ask third colleague', - }, - new AbortController().signal, - undefined, - secondSource, - ); - const thirdChannel = c!.channels.list().find((channel) => channel.remoteBot)!; - expect(c!.channels.readMessages(thirdChannel.id)[0]?.botCausation).toEqual({ - rootSourceEventId: root, - parentSourceEventId: secondSource, - hop: 2, - }); - await query(2); - let previousBody = 'Ask third colleague'; - for (let hop = 3; hop <= 33; hop++) { - const from = hop % 2 === 1 ? 2 : 1; - const to = from === 2 ? 1 : 2; - const source = inbound(from, previousBody); - const body = `Cycle hop ${hop}`; - await cores[from]!.runtime.runDirectBotTool!( - from === 2 ? 'bot' : 'replacement', + expect((await query(1, 'replacement')).result).toMatchObject({ + contacts: expect.arrayContaining([expect.objectContaining({ botId: 'remote:c:bot' })]), + }); + const secondSource = inbound(1, 'New identity'); + await b!.runtime.runDirectBotTool!( + 'replacement', 'bot_send_message', - { bot_id: to === 1 ? 'remote:b:replacement' : 'remote:c:bot', body }, + { + bot_id: 'remote:c:bot', + body: 'Ask third colleague', + }, new AbortController().signal, undefined, - source, + secondSource, ); - const channel = cores[to]!.channels.list().find( - (item) => - item.remoteBot?.hostId === nodes[from]!.nodeId && - item.members.includes(to === 1 ? 'replacement' : 'bot'), - )!; - const message = cores[to]!.channels.readMessages(channel.id).find( - (item) => item.body === body, - )!; - expect(message.botCausation).toEqual({ + const thirdChannel = c!.channels.list().find((channel) => channel.remoteBot)!; + expect(c!.channels.readMessages(thirdChannel.id)[0]?.botCausation).toEqual({ rootSourceEventId: root, - parentSourceEventId: source, - hop, + parentSourceEventId: secondSource, + hop: 2, }); - if (hop === 33) expect(message.deliveries).toBeUndefined(); - else - expect(message.deliveries).toEqual( - expect.arrayContaining([ - expect.objectContaining({ botSlug: to === 1 ? 'replacement' : 'bot' }), - ]), + await query(2); + let previousBody = 'Ask third colleague'; + for (let hop = 3; hop <= 33; hop++) { + const from = hop % 2 === 1 ? 2 : 1; + const to = from === 2 ? 1 : 2; + const source = inbound(from, previousBody); + const body = `Cycle hop ${hop}`; + await cores[from]!.runtime.runDirectBotTool!( + from === 2 ? 'bot' : 'replacement', + 'bot_send_message', + { bot_id: to === 1 ? 'remote:b:replacement' : 'remote:c:bot', body }, + new AbortController().signal, + undefined, + source, ); - previousBody = body; - } - await expect(send(0, root, 'remote:c:bot', 'Do not transitively enroll')).rejects.toMatchObject( - { code: 'unable-to-deliver' }, - ); - expect(a!.channels.list().some((channel) => channel.remoteBot?.hostId === 'c')).toBe(false); - } finally { - for (const bridge of bridges) await bridge.close(); - for (const core of cores) { - await core.runtime.close(); - core.operationalDatabase.close(); + const channel = cores[to]!.channels.list().find( + (item) => + item.remoteBot?.hostId === nodes[from]!.nodeId && + item.members.includes(to === 1 ? 'replacement' : 'bot'), + )!; + const message = cores[to]!.channels.readMessages(channel.id).find( + (item) => item.body === body, + )!; + expect(message.botCausation).toEqual({ + rootSourceEventId: root, + parentSourceEventId: source, + hop, + }); + if (hop === 33) expect(message.deliveries).toBeUndefined(); + else + expect(message.deliveries).toEqual( + expect.arrayContaining([ + expect.objectContaining({ botSlug: to === 1 ? 'replacement' : 'bot' }), + ]), + ); + previousBody = body; + } + await expect( + send(0, root, 'remote:c:bot', 'Do not transitively enroll'), + ).rejects.toMatchObject({ code: 'unable-to-deliver' }); + expect(a!.channels.list().some((channel) => channel.remoteBot?.hostId === 'c')).toBe(false); + } finally { + for (const bridge of bridges) await bridge.close(); + for (const core of cores) { + await core.runtime.close(); + core.operationalDatabase.close(); + } } - } -}, 120_000); + }, + 120_000, +); diff --git a/packages/core/test/network-withdrawal.test.ts b/packages/core/test/network-withdrawal.test.ts index cba1b001c6..5309e88799 100644 --- a/packages/core/test/network-withdrawal.test.ts +++ b/packages/core/test/network-withdrawal.test.ts @@ -13,8 +13,11 @@ import { receiveNetworkBotDm } from '../src/runtime/bot-network.js'; import { createDshBotAgentAdapter } from '../src/runtime/dsh-bot-agent-adapter.js'; import { FakeAgentHost } from './dsh-agent-host-fixture.js'; import { createTempRoot } from './helpers.js'; +import { loopbackAliasesAvailable } from './loopback-aliases.js'; -it.each(['pending', 'started', 'alternate-network', 'connect-route'] as const)( +const aliases = await loopbackAliasesAvailable(['127.0.0.12', '127.0.0.13']); + +it.skipIf(!aliases).each(['pending', 'started', 'alternate-network', 'connect-route'] as const)( 'carries a native removal through the real bridge and Inbox for %s work', async (phase) => { const nodes = [