diff --git a/HOST_INTEGRATION.md b/HOST_INTEGRATION.md index 0861c29..578cb56 100644 --- a/HOST_INTEGRATION.md +++ b/HOST_INTEGRATION.md @@ -46,7 +46,14 @@ GET /api/plugins/articraft/generations/:jobId POST /api/plugins/articraft/generations/:jobId/cancel ``` -Catalog responses use `ArticraftCatalogResponse`. Generation creation accepts +Catalog responses use `ArticraftCatalogResponse`. Each item's `dimensions` is its +rest-pose size in Pascal's frame (Y up, metres) and its optional `offset` stands +the asset on its bottom centre at the node origin; placement copies both onto the +node. A host that syncs a catalog computes them with `urdfRestBounds` (every +visual through its link, joint and visual origins and mesh scale) and +`bottomCentreFrame`, both exported by the package. + +Generation creation accepts multipart form data (`prompt`, optional `image`, optional `provider`, optional `model`) and returns `{ id, status }`. Configuration returns the worker-selected default plus its allowlisted `models`; OpenAI/GPT-5.6 remains selected when those diff --git a/src/bounds.test.ts b/src/bounds.test.ts new file mode 100644 index 0000000..c13d837 --- /dev/null +++ b/src/bounds.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, test } from 'bun:test' +import { bottomCentreFrame, objExtent, urdfRestBounds } from './index' + +// A 1 × 0.5 × 0.2 m base standing on z = 0, a lid hinged on its top edge +// (joint origin rotated a quarter turn about Z) carrying a Z-up cylinder, and +// a handle modelled in millimetres (mesh scale 0.001). +const URDF = ` + + + + + + + + + + + + + + + + + + + + + + + + + + + + +` + +const handle = objExtent('# handle\nv 0 -10 0\nv 100 10 20\nvn 0 0 1\nf 1 2 1\n') + +describe('rest-pose bounds', () => { + test('reads an OBJ vertex extent', () => { + expect(handle).toEqual({ min: [0, -10, 0], max: [100, 10, 20] }) + expect(objExtent('# no vertices\n')).toBeNull() + }) + + test('counts every visual with link, joint and visual origins and mesh scale, Y up', async () => { + const bounds = await urdfRestBounds(URDF, async (filename) => + filename === 'assets/handle.obj' ? handle : null, + ) + // URDF frame: base x ±0.5, y ±0.25, z 0..0.2. The lid frame sits at + // (0.2, 0, 0.2) turned 90° about Z: its cylinder spans x 0.1..0.3, + // y -0.1..0.1, z 0.2..0.6; its handle (0.3 m along lid x = world y) + // spans x 0.19..0.21, y 0.3..0.4, z 0.2..0.22. + // Pascal frame (x, z, -y): x -0.5..0.5, y 0..0.6, z -0.4..0.25. + expect(bounds).not.toBeNull() + const expected = { min: [-0.5, 0, -0.4], max: [0.5, 0.6, 0.25] } + for (const key of ['min', 'max'] as const) { + for (const axis of [0, 1, 2]) { + expect(bounds?.[key][axis]).toBeCloseTo(expected[key][axis] ?? Number.NaN, 9) + } + } + }) + + test('returns null when nothing is visible', async () => { + expect(await urdfRestBounds('', async () => null)).toBeNull() + }) + + test('stands an asset that extends below its origin on its bottom centre', () => { + const frame = bottomCentreFrame({ min: [-0.2, -0.35, 0.1], max: [0.6, 0.45, 0.5] }) + expect(frame.dimensions).toEqual([0.8, 0.8, 0.4]) + expect(frame.offset).toEqual([-0.2, 0.35, -0.3]) + }) +}) diff --git a/src/bounds.ts b/src/bounds.ts new file mode 100644 index 0000000..5982320 --- /dev/null +++ b/src/bounds.ts @@ -0,0 +1,227 @@ +import type { Vec3 } from './types' + +/** Axis-aligned bounds in the frame Pascal renders an asset in: Y up, metres. */ +export type Bounds = { min: Vec3; max: Vec3 } + +/** Where a mesh's vertices lie in its own frame, before the visual's `scale`. */ +export type MeshExtent = { min: Vec3; max: Vec3 } + +type XmlElement = { name: string; attributes: Record; children: XmlElement[] } +type Matrix = number[] // 4 × 4, row-major + +const IDENTITY: Matrix = [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1] + +/** + * The rest-pose bounds of a URDF (every joint at 0) in Pascal's frame: the + * URDF's Z-up axes turned Y-up, as the renderer turns the loaded robot + * (`rotation.x = -π/2`). Every visual counts (box, cylinder, sphere, mesh) + * through its link, joint and visual origins and the mesh `scale`; a + * rotated visual contributes the bounds of its turned box. `meshExtent` + * resolves a mesh `filename` as written in the URDF. Null when nothing is + * visible. + */ +export async function urdfRestBounds( + urdf: string, + meshExtent: (filename: string) => Promise, +): Promise { + const robot = parseXml(urdf).find((element) => element.name === 'robot') + if (!robot) return null + const links = robot.children.filter((element) => element.name === 'link') + const jointByChild = new Map() + for (const joint of robot.children.filter((element) => element.name === 'joint')) { + const child = childNamed(joint, 'child')?.attributes.link + if (child) jointByChild.set(child, joint) + } + const worldByLink = new Map() + const world = (link: string, visiting = new Set()): Matrix => { + const known = worldByLink.get(link) + if (known) return known + const joint = jointByChild.get(link) + const parent = joint && childNamed(joint, 'parent')?.attributes.link + // A root link, or a cycle the URDF should not have, sits at the origin. + const matrix = + joint && parent && !visiting.has(parent) + ? multiply(world(parent, visiting.add(link)), originOf(joint)) + : IDENTITY + worldByLink.set(link, matrix) + return matrix + } + + const min: Vec3 = [Infinity, Infinity, Infinity] + const max: Vec3 = [-Infinity, -Infinity, -Infinity] + for (const link of links) { + const linkWorld = world(link.attributes.name ?? '') + for (const visual of link.children.filter((element) => element.name === 'visual')) { + const extent = await visualExtent(visual, meshExtent) + if (!extent) continue + const matrix = multiply(linkWorld, originOf(visual)) + for (const x of [extent.min[0], extent.max[0]]) { + for (const y of [extent.min[1], extent.max[1]]) { + for (const z of [extent.min[2], extent.max[2]]) { + const point = transform(matrix, [x, y, z]) + for (const axis of [0, 1, 2] as const) { + min[axis] = Math.min(min[axis], point[axis]) + max[axis] = Math.max(max[axis], point[axis]) + } + } + } + } + } + } + if (!min.every(Number.isFinite)) return null + // URDF (x, y, z) Z up → Pascal (x, z, -y) Y up. + return { min: [min[0], min[2], -max[1]], max: [max[0], max[2], -min[1]] } +} + +/** + * The footprint and the offset that stand an asset on its bottom centre at the + * node's origin: the renderer moves the loaded asset by `offset` (before the + * node's scale), so its rest pose rests on the floor, centred on the node, + * like the placeholder box. Rounded to 0.1 mm. + */ +export function bottomCentreFrame(bounds: Bounds): { dimensions: Vec3; offset: Vec3 } { + const [minX, minY, minZ] = bounds.min + const [maxX, maxY, maxZ] = bounds.max + return { + dimensions: [ + round(Math.max(maxX - minX, 0.001)), + round(Math.max(maxY - minY, 0.001)), + round(Math.max(maxZ - minZ, 0.001)), + ], + offset: [round(-(minX + maxX) / 2), round(-minY), round(-(minZ + maxZ) / 2)], + } +} + +/** The extent of an OBJ's `v x y z` vertices, null when it has none. */ +export function objExtent(obj: string): MeshExtent | null { + const min: Vec3 = [Infinity, Infinity, Infinity] + const max: Vec3 = [-Infinity, -Infinity, -Infinity] + for (const line of obj.split('\n')) { + if (!line.startsWith('v ')) continue + const values = line.trim().split(/\s+/).slice(1, 4).map(Number) + if (values.length !== 3 || !values.every(Number.isFinite)) continue + for (const axis of [0, 1, 2] as const) { + min[axis] = Math.min(min[axis], values[axis] as number) + max[axis] = Math.max(max[axis], values[axis] as number) + } + } + return min.every(Number.isFinite) ? { min, max } : null +} + +async function visualExtent( + visual: XmlElement, + meshExtent: (filename: string) => Promise, +): Promise { + const shape = childNamed(visual, 'geometry')?.children[0] + if (!shape) return null + const number = (name: string) => Number(shape.attributes[name]) + if (shape.name === 'box') { + const [x, y, z] = vector(shape.attributes.size, [0, 0, 0]) + return { min: [-x / 2, -y / 2, -z / 2], max: [x / 2, y / 2, z / 2] } + } + if (shape.name === 'cylinder') { + const radius = number('radius') + const half = number('length') / 2 + if (!(Number.isFinite(radius) && Number.isFinite(half))) return null + return { min: [-radius, -radius, -half], max: [radius, radius, half] } + } + if (shape.name === 'sphere') { + const radius = number('radius') + return Number.isFinite(radius) + ? { min: [-radius, -radius, -radius], max: [radius, radius, radius] } + : null + } + if (shape.name === 'mesh' && shape.attributes.filename) { + const extent = await meshExtent(shape.attributes.filename) + if (!extent) return null + const scale = vector(shape.attributes.scale, [1, 1, 1]) + const a = extent.min.map((value, axis) => value * (scale[axis] ?? 1)) as Vec3 + const b = extent.max.map((value, axis) => value * (scale[axis] ?? 1)) as Vec3 + return { + min: [Math.min(a[0], b[0]), Math.min(a[1], b[1]), Math.min(a[2], b[2])], + max: [Math.max(a[0], b[0]), Math.max(a[1], b[1]), Math.max(a[2], b[2])], + } + } + return null +} + +/** A URDF `` (fixed-axis roll, pitch, yaw) as a matrix. */ +function originOf(element: XmlElement): Matrix { + const origin = childNamed(element, 'origin') + const [x, y, z] = vector(origin?.attributes.xyz, [0, 0, 0]) + const [roll, pitch, yaw] = vector(origin?.attributes.rpy, [0, 0, 0]) + const [cr, sr, cp, sp, cy, sy] = [ + Math.cos(roll), + Math.sin(roll), + Math.cos(pitch), + Math.sin(pitch), + Math.cos(yaw), + Math.sin(yaw), + ] + // R = Rz(yaw) · Ry(pitch) · Rx(roll) + return [ + cy * cp, cy * sp * sr - sy * cr, cy * sp * cr + sy * sr, x, + sy * cp, sy * sp * sr + cy * cr, sy * sp * cr - cy * sr, y, + -sp, cp * sr, cp * cr, z, + 0, 0, 0, 1, + ] +} + +function multiply(a: Matrix, b: Matrix): Matrix { + const out: Matrix = new Array(16).fill(0) + for (let row = 0; row < 4; row += 1) { + for (let column = 0; column < 4; column += 1) { + let sum = 0 + for (let k = 0; k < 4; k += 1) sum += (a[row * 4 + k] ?? 0) * (b[k * 4 + column] ?? 0) + out[row * 4 + column] = sum + } + } + return out +} + +function transform(m: Matrix, [x, y, z]: Vec3): Vec3 { + const at = (index: number) => m[index] ?? 0 + return [ + at(0) * x + at(1) * y + at(2) * z + at(3), + at(4) * x + at(5) * y + at(6) * z + at(7), + at(8) * x + at(9) * y + at(10) * z + at(11), + ] +} + +function vector(value: string | undefined, fallback: Vec3): Vec3 { + const parts = value?.trim().split(/\s+/).map(Number) + return parts?.length === 3 && parts.every(Number.isFinite) ? (parts as Vec3) : fallback +} + +function childNamed(element: XmlElement, name: string): XmlElement | undefined { + return element.children.find((child) => child.name === name) +} + +function round(value: number): number { + return Math.round(value * 10_000) / 10_000 + 0 +} + +/** Elements of a small XML document (URDF): declarations and comments skipped. */ +function parseXml(source: string): XmlElement[] { + const roots: XmlElement[] = [] + const stack: XmlElement[] = [] + const tag = /|<\?[\s\S]*?\?>||]*>|<\/\s*([\w:.-]+)\s*>|<([\w:.-]+)((?:\s+[\w:.-]+\s*=\s*(?:"[^"]*"|'[^']*'))*)\s*(\/?)>/g + for (const match of source.matchAll(tag)) { + const [, closing, name, rawAttributes, selfClosing] = match + if (closing) { + stack.pop() + continue + } + if (!name) continue + const attributes: Record = {} + for (const attribute of (rawAttributes ?? '').matchAll(/([\w:.-]+)\s*=\s*(?:"([^"]*)"|'([^']*)')/g)) { + attributes[attribute[1] as string] = attribute[2] ?? attribute[3] ?? '' + } + const element: XmlElement = { name, attributes, children: [] } + const parent = stack.at(-1) + if (parent) parent.children.push(element) + else roots.push(element) + if (!selfClosing) stack.push(element) + } + return roots +} diff --git a/src/index.ts b/src/index.ts index e764045..01c93ee 100644 --- a/src/index.ts +++ b/src/index.ts @@ -27,6 +27,8 @@ export const articraftHostPanel: EditorHostPanel = { } export { ARTICRAFT_API_BASE, fetchCatalog } from './api' +export { bottomCentreFrame, objExtent, urdfRestBounds } from './bounds' +export type { Bounds, MeshExtent } from './bounds' export { articraftAssetDefinition } from './definition' export { ArticraftAssetNode } from './schema' export type { diff --git a/src/renderer.tsx b/src/renderer.tsx index c0c7dd0..fcab1ac 100644 --- a/src/renderer.tsx +++ b/src/renderer.tsx @@ -180,7 +180,13 @@ export function ArticraftVisual({ useEffect(() => () => fallbackMaterial.dispose(), [fallbackMaterial]) - if (loaded) return + if (loaded) { + return ( + + + + ) + } return ( { test('persists an immutable artifact and joint pose', () => { @@ -38,6 +38,11 @@ describe('Articraft asset schema', () => { expect(node.jointValues.globe_spin).toBe(1.25) expect(node.motionEnabled).toBe(true) expect(node.artifact.sha256).toBe('a'.repeat(64)) + const placed = ArticraftAssetNode.parse({ ...node, offset: [0.01, 0.12, -0.02] }) + const reopened = ArticraftAssetNode.parse(JSON.parse(JSON.stringify(placed))) + expect(reopened).toEqual(placed) + expect(reopened.offset).toEqual([0.01, 0.12, -0.02]) + expect(reopened.motionEnabled).toBe(true) }) test('keeps older nodes still until motion is explicitly enabled', () => { @@ -54,5 +59,6 @@ describe('Articraft asset schema', () => { }) expect(node.motionEnabled).toBe(false) + expect(node.offset).toBeUndefined() }) }) diff --git a/src/schema.ts b/src/schema.ts index 970f4f1..88ddfa2 100644 --- a/src/schema.ts +++ b/src/schema.ts @@ -38,6 +38,8 @@ export const ArticraftAssetNode = BaseNode.extend({ }), thumbnailUrl: z.string().url().optional(), dimensions: Vec3.default([1, 1, 1]), + /** Moves the loaded asset so its rest pose stands on its bottom centre at the node origin. */ + offset: Vec3.optional(), parts: z .array( z.object({ diff --git a/src/tool.tsx b/src/tool.tsx index 4ab685b..e45781c 100644 --- a/src/tool.tsx +++ b/src/tool.tsx @@ -26,6 +26,7 @@ export default function ArticraftTool() { artifact: item.artifact, thumbnailUrl: item.thumbnailUrl, dimensions: item.dimensions, + offset: item.offset, parts: item.parts, joints: item.joints, jointValues: item.defaultJointValues, @@ -49,6 +50,7 @@ export default function ArticraftTool() { artifact: item.artifact, thumbnailUrl: item.thumbnailUrl, dimensions: item.dimensions, + offset: item.offset, parts: item.parts, joints: item.joints, jointValues: item.defaultJointValues, diff --git a/src/types.ts b/src/types.ts index 40829cc..410fa6f 100644 --- a/src/types.ts +++ b/src/types.ts @@ -57,7 +57,13 @@ export type ArticraftCatalogItem = { source: ArticraftSource artifact: ArticraftArtifact thumbnailUrl?: string + /** Rest-pose size in Pascal's frame (Y up, metres). */ dimensions: Vec3 + /** + * Moves the loaded asset so its rest pose stands on its bottom centre at the + * node origin (`bottomCentreFrame`). Absent in records synced before it existed. + */ + offset?: Vec3 parts: ArticraftPart[] joints: ArticraftJoint[] defaultJointValues: Record