diff --git a/test/backend/test_graph.py b/test/backend/test_graph.py index 3d5ac85e2e044..0adff33178e1e 100644 --- a/test/backend/test_graph.py +++ b/test/backend/test_graph.py @@ -56,6 +56,11 @@ def run_schedule(calls:list[UOp]): def zero_bufs(bufs): for b in bufs: b.copy_from(Buffer("PYTHON", b.size, b.dtype, opaque=memoryview(bytearray(b.nbytes)))) +def fill_int_buf(buf:Buffer, values:np.ndarray): + values = np.asarray(values, dtype=np.int32) + assert values.size == buf.size + buf.copy_from(Buffer("PYTHON", buf.size, buf.dtype, opaque=memoryview(bytearray(values.tobytes())))) + @unittest.skipUnless(Device[Device.DEFAULT].graph is not None, "graph support required") class TestGraph(unittest.TestCase): def skip_if_no_offset(self): @@ -66,6 +71,35 @@ def skip_if_not_multigraph(self): if not issubclass(graph, MultiGraphRunner): self.skipTest("graph is not supported (not MultiGraphRunner)") if not hasattr(d.allocator, '_transfer') or not d.allocator.supports_transfer: self.skipTest("device is not supported (no transfers)") + @needs_second_gpu + def test_copy_graph_rebinds_nonzero_views(self): + self.skip_if_not_multigraph() + self.skip_if_no_offset() + d0, d1, base_size, copy_size = Device.DEFAULT, f"{Device.DEFAULT}:1", 16, 4 + dstp, srcp = UOp.param(0, dtypes.int, base_size, d0), UOp.param(1, dtypes.int, base_size, d1) + copy = UOp(Ops.COPY, src=(UOp.param(1, dtypes.int, copy_size, d1),), arg=d0) + call = copy.call(dstp.shrink(((3, 3+copy_size),)), srcp.shrink(((5, 5+copy_size),))) + graph_call = UOp(Ops.CUSTOM_FUNCTION, src=(UOp(Ops.LINEAR, src=(call,)),), arg="graph") + + initial_dst, initial_src = make_buffer(d0, base_size), make_buffer(d1, base_size) + initial = np.full(base_size, -1, dtype=np.int32) + fill_int_buf(initial_dst, initial) + fill_int_buf(initial_src, np.arange(base_size, dtype=np.int32)) + graph = Device[d0].graph(graph_call, (UOp.from_buffer(initial_dst), UOp.from_buffer(initial_src))) + + dst, src = make_buffer(d0, base_size), make_buffer(d1, base_size) + dstv, srcv = np.full(base_size, -2, dtype=np.int32), np.arange(base_size, dtype=np.int32) + 100 + fill_int_buf(dst, dstv) + fill_int_buf(src, srcv) + graph((UOp.from_buffer(dst), UOp.from_buffer(src)), {}) + Device[d0].synchronize() + Device[d1].synchronize() + + expected = dstv.copy() + expected[3:3+copy_size] = srcv[5:5+copy_size] + np.testing.assert_equal(expected, np.frombuffer(dst.as_memoryview(), np.int32)) + np.testing.assert_equal(initial, np.frombuffer(initial_dst.as_memoryview(), np.int32)) + def test_order_2_writes_to_same_buf(self): d0 = Device.DEFAULT b = [make_buffer(d0, fill=True) for _ in range(5)] @@ -315,4 +349,4 @@ def test_middle_write_splits_write_dep(self): for buf in outs: np.testing.assert_equal(expected[buf], np.frombuffer(buf.as_memoryview(), np.int32)) if __name__ == '__main__': - unittest.main() + unittest.main() \ No newline at end of file diff --git a/test/unit/test_jit.py b/test/unit/test_jit.py index 428bdb05ec3b5..7d069db4e9f1f 100644 --- a/test/unit/test_jit.py +++ b/test/unit/test_jit.py @@ -12,6 +12,23 @@ def _simple_test(add, extract=lambda x: x, N=10): assert_jit_cache_len(add, 1) class TestJit(unittest.TestCase): + def test_graph_rebinds_copy_views(self): + from tinygrad.engine.jit import GraphRunner + from tinygrad.uop.ops import Ops + dst, src = UOp.param(0, dtypes.int32, 6, "CPU"), UOp.param(1, dtypes.int32, 5, "CPU:1") + copy = UOp(Ops.COPY, src=(UOp.param(1, dtypes.int32, 2, "CPU:1"),), arg="CPU") + call = copy.call(dst.shrink(((2, 4),)), src.shrink(((1, 3),))) + d0, s0 = Tensor.zeros(6, dtype=dtypes.int32, device="CPU").realize(), Tensor.zeros(5, dtype=dtypes.int32, device="CPU:1").realize() + graph = GraphRunner(UOp(Ops.CALL, src=(UOp(Ops.LINEAR, src=(call,)),)), (d0.uop, s0.uop)) + d1 = Tensor.zeros(6, dtype=dtypes.int32, device="CPU").realize() + s1 = Tensor([10, 20, 30, 40, 50], device="CPU:1").realize() + replacements = dict(graph.updated_buffers(0, (d1.uop, s1.uop))) + self.assertIs(replacements[0].base, d1.uop.buffer.base) + self.assertIs(replacements[1].base, s1.uop.buffer.base) + self.assertEqual((replacements[0].offset, replacements[1].offset), (8, 4)) + replacements[0].copy_from(replacements[1]) + self.assertListEqual(d1.tolist(), [0, 0, 20, 30, 0, 0]) + def test_jitbeam_triggers_beam(self): from unittest.mock import patch from tinygrad.helpers import getenv as _getenv @@ -409,6 +426,15 @@ def w2(x) -> Tensor: return (weights*2).contiguous() + x np.testing.assert_allclose(out.tolist(), [x*2+y for x,y in zip(weights.tolist(), a.tolist())]) assert_jit_cache_len(w2_prune, 1) + def test_prune_slice_assign_from_fixed_source(self): + src = Tensor([7, 8], device="CPU:1").realize() + @TinyJit(prune=True) + def write(dst): dst[1:3].assign(src.to(dst.device)).realize() + for fill in range(4): + dst = Tensor.full(4, fill, dtype=dtypes.int32, device="CPU").realize() + write(dst) + self.assertListEqual(dst.tolist(), [fill, 7, 8, fill]) + class TestJitInsideJit(unittest.TestCase): def test_jit_jit_error(self): @@ -439,4 +465,4 @@ def f(): return Tensor.randint(20, high=5) self.assertListEqual(t0, t1, msg=f"mismatch at list {i}") if __name__ == '__main__': - unittest.main() + unittest.main() \ No newline at end of file diff --git a/test/unit/test_setitem_schedule.py b/test/unit/test_setitem_schedule.py index 90885316bc7c9..e38f17b3ba9fb 100644 --- a/test/unit/test_setitem_schedule.py +++ b/test/unit/test_setitem_schedule.py @@ -1,5 +1,7 @@ import unittest -from tinygrad import Tensor, Device, dtypes, GlobalCounters +from tinygrad import Tensor, Device, TinyJit, Variable, dtypes, GlobalCounters +from tinygrad.engine.realize import run_linear +from tinygrad.uop.ops import Ops from test.helpers import assert_kernel_count class TestSetitemInto(unittest.TestCase): @@ -141,14 +143,73 @@ def test_setitem_full(self): @unittest.skipUnless(Device.DEFAULT != "CPU", "source must be on another device") def test_setitem_slice_assign_from_other_device(self): - # NOTE: this is 2 kernels, the cross-device copy should fuse with the assign into one a = Tensor.ones(20, device="CPU") b = Tensor.arange(20).float().clone() Tensor.realize(a, b) GlobalCounters.reset() a[10:12].assign(b[13:15].to(a.device)).realize() - assert_kernel_count(2) + assert_kernel_count(2 if Device.DEFAULT.startswith(("PYTHON", "NPY", "DISK", "CL", "WEBGPU", "NULL")) else 1) self.assertListEqual(a.tolist(), [1.0]*10 + [13.0, 14.0] + [1.0]*8) + def test_cross_device_slice_copy_targets_destination_view(self): + dst = Tensor(list(range(8)), dtype=dtypes.int32, device="CPU:1").realize() + src = Tensor(list(range(10, 18)), dtype=dtypes.int32, device="CPU").realize() + linear = dst[6:8].assign(src[3:5].to(dst.device)).schedule_linear() + self.assertEqual(len(linear.src), 1) + call = linear.src[0] + self.assertIs(call.src[0].op, Ops.COPY) + self.assertEqual((call.src[1].contiguous_view_offset(), call.src[2].contiguous_view_offset()), (6, 3)) + self.assertEqual(call.src[1].max_numel(), 2) + run_linear(linear) + self.assertListEqual(dst.tolist(), [0, 1, 2, 3, 4, 5, 13, 14]) + + def test_cross_device_contiguous_reshape_uses_single_copy(self): + dst = Tensor(list(range(30)), device="CPU").realize() + src = Tensor(list(range(100, 110)), device="CPU:1").reshape(2, 5).realize() + linear = dst.reshape(6, 5)[2:4, :].assign(src.to(dst.device)).schedule_linear() + self.assertEqual([x.src[0].op for x in linear.src], [Ops.COPY]) + run_linear(linear) + self.assertListEqual(dst.tolist(), list(range(10)) + list(range(100, 110)) + list(range(20, 30))) + + def test_cross_device_noncontiguous_destination_falls_back(self): + dst = Tensor(list(range(12)), device="CPU").realize() + src = Tensor(list(range(100, 104)), device="CPU:1").realize() + linear = dst[1:9:2].assign(src.to(dst.device)).schedule_linear() + self.assertGreaterEqual(len(linear.src), 2) + run_linear(linear) + self.assertListEqual(dst.tolist(), [0, 100, 2, 101, 4, 102, 6, 103, 8, 9, 10, 11]) + + def test_cross_device_noncontiguous_source_falls_back(self): + dst = Tensor.zeros(2, dtype=dtypes.int32, device="CPU").realize() + src = Tensor(list(range(8)), dtype=dtypes.int32, device="CPU:1").reshape(2, 4).T[1] + linear = dst.assign(src.to(dst.device)).schedule_linear() + self.assertGreaterEqual(len(linear.src), 2) + run_linear(linear) + self.assertListEqual(dst.tolist(), [1, 5]) + + def test_cross_device_shared_copy_falls_back(self): + dst = Tensor.zeros(6, dtype=dtypes.int32, device="CPU").realize() + rhs = Tensor([7, 8], device="CPU:1").realize().to(dst.device) + dst[0:2].assign(rhs) + linear = dst[4:6].assign(rhs).schedule_linear() + self.assertGreaterEqual(len(linear.src), 2) + run_linear(linear) + self.assertListEqual(dst.tolist(), [7, 8, 0, 0, 7, 8]) + self.assertListEqual(rhs.tolist(), [7, 8]) + + def test_symbolic_length_cross_device_slice_does_not_overcopy(self): + @TinyJit + def write(dst:Tensor, src:Tensor, v): + dst.shrink(((7, 7+v),)).assign(src.shrink(((9, 9+v),)).to(dst.device)).realize() + for i, value in enumerate((1, 4, 2)): + before = [-100-i]*20 + values = list(range(100*i, 100*i+24)) + dst = Tensor(before, dtype=dtypes.int32, device="CPU").realize() + src = Tensor(values, dtype=dtypes.int32, device="CPU:1").realize() + write(dst, src, Variable("slice_len", 1, 4).bind(value)) + expected = before[:] + expected[7:7+value] = values[9:9+value] + self.assertListEqual(dst.tolist(), expected) + if __name__ == '__main__': - unittest.main() + unittest.main() \ No newline at end of file diff --git a/tinygrad/engine/jit.py b/tinygrad/engine/jit.py index 72fbd9ac3d16a..dd91f91b9fd5a 100644 --- a/tinygrad/engine/jit.py +++ b/tinygrad/engine/jit.py @@ -7,7 +7,7 @@ from tinygrad.uop.ops import UOp, PatternMatcher, Variable, sym_infer, Ops, rewrite_group, graph_rewrite from tinygrad.renderer import Estimates from tinygrad.engine.realize import capturing, compile_linear, link_linear, run_linear, graph_cache, estimate_uop, get_runtime -from tinygrad.engine.realize import unwrap_multi, resolve_params, get_call_arg_uops, get_call_written_bufs +from tinygrad.engine.realize import unwrap_multi, resolve_params, get_call_arg_uops, get_call_written_bufs, _resolve from tinygrad.schedule.memory import memory_plan_rewrite, _collect_bufs from tinygrad.nn.state import get_parameters from tinygrad.uop.movement import mop_cleanup @@ -16,7 +16,7 @@ def prune_linear(linear:UOp, needed:set[UOp]) -> tuple[UOp, UOp]: kept, onetime = [], [] for si in linear.src: - si_bufs = {b for src in si.src[1:] for b in _collect_bufs(src)} + si_bufs = {b for src in si.src[1:] for b in _collect_bufs(src.storage_base)} if not si_bufs.isdisjoint(needed): kept.append(si) needed |= si_bufs @@ -91,13 +91,14 @@ def __init__(self, linear:UOp, input_uops:tuple[UOp, ...]=()): self.linear = linear.src[0] self.calls: list[tuple[int, UOp, list[Buffer], dict[str, int]]] = [] self.runtimes: list[Any|None] = [] - self.uop_replace: list[list[tuple[int, int]]] = [] + self.uop_replace: list[list[tuple[int, UOp]]] = [] for call in self.linear.src: - replace = [(p, b.arg.slot) for p, b in enumerate(get_call_arg_uops(call)) if b.op is Ops.PARAM] + buffer_replace = [(p, b) for p, b in enumerate(get_call_arg_uops(call)) + if b.op is Ops.PARAM or (call.src[0].op is Ops.COPY and _resolve(b, input_uops) is not b)] for dev_idx, (bufs, device_vars) in enumerate(unwrap_multi(call, resolve_params(call, input_uops))): self.calls.append((dev_idx, call.src[0], [b.ensure_allocated() for b in bufs], device_vars)) self.runtimes.append(get_runtime(bufs[0].device, call.src[0]) if call.src[0].op is Ops.PROGRAM else None) - self.uop_replace.append(replace) + self.uop_replace.append(buffer_replace) self.var_vals_replace:dict[int, list[tuple[int, int]]] = {} self.launch_dims_replace:dict[int, tuple[int|None, int|None]] = {} @@ -128,6 +129,11 @@ def find_symbolic_dim(dim:tuple[int,int,int]): return self.symbolic_dims.index(t def __call__(self, input_uops:tuple[UOp, ...], var_vals:dict[str, int], wait=False) -> float|None: raise NotImplementedError("override this") + def updated_buffers(self, j:int, input_uops:tuple[UOp, ...]): + for pos, u in self.uop_replace[j]: + buf = _resolve(u, input_uops).buffer + yield pos, (buf.bufs[self.calls[j][0]] if isinstance(buf, MultiBuffer) else buf).ensure_allocated() + def updated_vars(self, var_vals: dict[str, int]): vals = [var_vals[v] for v in self.vars] for j, vidxs in self.var_vals_replace.items(): @@ -291,4 +297,4 @@ def _buf_or_none(u:UOp) -> Buffer|MultiBuffer|None: return u.arg.buffer if u.op def TinyJit(fxn:Callable[..., ReturnType], *, prune:bool=False) -> _TinyJit[ReturnType]: ... @overload def TinyJit(fxn:None=None, *, prune:bool=False) -> Callable[[Callable[..., ReturnType]], _TinyJit[ReturnType]]: ... -def TinyJit(fxn=None, **kwargs): return (lambda f: _TinyJit(f, **kwargs)) if fxn is None else _TinyJit(fxn, **kwargs) +def TinyJit(fxn=None, **kwargs): return (lambda f: _TinyJit(f, **kwargs)) if fxn is None else _TinyJit(fxn, **kwargs) \ No newline at end of file diff --git a/tinygrad/runtime/graph/cuda.py b/tinygrad/runtime/graph/cuda.py index ed228d1350927..1869a264a7790 100644 --- a/tinygrad/runtime/graph/cuda.py +++ b/tinygrad/runtime/graph/cuda.py @@ -2,7 +2,7 @@ from typing import Any, cast import tinygrad.runtime.autogen.cuda as cuda from tinygrad.runtime.support.c import init_c_var -from tinygrad.device import Device, MultiBuffer +from tinygrad.device import Device from tinygrad.uop.ops import UOp, Ops from tinygrad.runtime.ops_cuda import CUDADevice, check, encode_args, cu_time_execution from tinygrad.engine.jit import MultiGraphRunner @@ -47,9 +47,8 @@ def new_node(self, bufs, write): def __call__(self, input_uops:tuple[UOp, ...], var_vals:dict[str, int], wait=False): # Update buffers in the c_args struct. for j in self.updatable: - (_, params, c_args, is_copy), dev_idx = self.nodes[j], self.calls[j][0] - for pos, iidx in self.uop_replace[j]: - buf = b.bufs[dev_idx] if isinstance(b:=input_uops[iidx].buffer, MultiBuffer) else b + _, params, c_args, is_copy = self.nodes[j] + for pos, buf in self.updated_buffers(j, input_uops): if not is_copy: setattr(c_args, f'f{pos}', buf._buf) else: setattr(params, 'srcDevice' if pos == 1 else 'dstDevice', buf._buf) diff --git a/tinygrad/runtime/graph/metal.py b/tinygrad/runtime/graph/metal.py index 80b99c4b73158..12d1eea74c011 100644 --- a/tinygrad/runtime/graph/metal.py +++ b/tinygrad/runtime/graph/metal.py @@ -1,7 +1,7 @@ from typing import Any, cast import ctypes, decimal, struct from tinygrad.helpers import dedup, getenv, unwrap, PROFILE -from tinygrad.device import Buffer, Device, ProfileGraphEntry, ProfileGraphEvent +from tinygrad.device import Device, ProfileGraphEntry, ProfileGraphEvent from tinygrad.uop.ops import UOp, Ops from tinygrad.engine.jit import GraphRunner, GraphException from tinygrad.runtime.ops_metal import MetalDevice, wait_check, to_ns_str @@ -61,8 +61,7 @@ def __call__(self, input_uops:tuple[UOp, ...], var_vals:dict[str, int], wait=Fal updated_bufs = [] for j in self.updatable: computeCommand = self.icb.indirectComputeCommandAtIndex(j) - for pos, iidx in self.uop_replace[j]: - buf = cast(Buffer, input_uops[iidx].buffer) + for pos, buf in self.updated_buffers(j, input_uops): computeCommand.setKernelBuffer_offset_atIndex(buf._buf.buf, buf._buf.offset, pos) updated_bufs.append(buf._buf.buf) diff --git a/tinygrad/schedule/__init__.py b/tinygrad/schedule/__init__.py index a79b1c2ef84fc..7a6269693a387 100644 --- a/tinygrad/schedule/__init__.py +++ b/tinygrad/schedule/__init__.py @@ -80,7 +80,7 @@ def create_schedule(sched_sink:UOp) -> UOp: from tinygrad.schedule.memory import memory_plan_rewrite from tinygrad.engine.realize import capturing, pm_flatten_linear -from tinygrad.schedule.prepare import prepare_rangeify +from tinygrad.schedule.prepare import prepare_rangeify, supports_sliced_copy_destination from tinygrad.schedule.rangeify import get_kernel_graph from tinygrad.helpers import CAPTURING from tinygrad.uop.ops import PatternMatcher, UPat, ParamArg @@ -152,9 +152,25 @@ def assert_all_same_devices(ast:UOp): devices = dedup([x.device for x in ast.toposort() if x.op is Ops.PARAM and x.device is not None]) if len(devices) >= 2: raise RuntimeError(f"all buffers must be on the same device: {devices}") -def copy_kernel_to_copy_uop(call:UOp, dst:UOp, src:UOp, r:UOp|None=None): +def copy_kernel_to_copy_uop(call:UOp, dst:UOp, src:UOp, r:UOp|None=None, offset:UOp|None=None): if dst.device == src.device and not (isinstance(dst.device, str) and dst.device.startswith("DISK")): return None - return call.replace(src=(UOp(Ops.COPY, src=(src,), arg=dst.device),) + call.src[1:]) + # A runtime COPY has a fixed byte count. The RANGE must therefore cover the exact concrete source PARAM. + if r is not None and (len(r.src) != 1 or r.src[0].op is not Ops.CONST or r.src[0].val != src.arg.size): return None + copy = UOp(Ops.COPY, src=(src,), arg=dst.device) + if offset is None and (dst.arg.size == src.arg.size or not supports_sliced_copy_destination(dst)): + return call.replace(src=(copy,) + call.src[1:]) + if offset is None: offset = UOp.const(0) + if (not supports_sliced_copy_destination(dst) or not isinstance(offset.val, int) or + dst.arg.slot != 0 or src.arg.slot != 1 or len(call.src) != 3 or + not isinstance(dst.device, str) or not isinstance(src.device, str) or + not isinstance(src.arg.size, int) or src.arg.size <= 0 or dst.dtype != src.dtype): return None + dest, source, size = call.src[1], call.src[2], src.arg.size + if (dest.dtype != dst.dtype or source.dtype != src.dtype or dest.max_numel() != dst.arg.size or source.max_numel() != size or + offset.val < 0 or offset.val + size > dest.max_numel()): return None + dest = dest.shrink(((offset.val, offset.val+size),)) + if ((dview:=dest.contiguous_view()) is None or source.contiguous_view() is None or + dview[0].storage_base is not call.src[1].storage_base or dview[1] != offset.val): return None + return call.replace(src=(copy, dest, source)) def simplify_copy_kernel(call:UOp, ast:UOp, dst:UOp, src:UOp): # NOTE: this is a codegen for SDMA devices @@ -176,6 +192,12 @@ def simplify_copy_kernel(call:UOp, ast:UOp, dst:UOp, src:UOp): (UPat(Ops.CALL, src=(UPat(Ops.PARAM, name="dst").index(UPat(Ops.RANGE, name="r")) .store(UPat(Ops.PARAM, name="src").index(UPat(Ops.RANGE, name="r"))).end(UPat(Ops.RANGE, name="r")).sink(),), name="call", allow_any_len=True), copy_kernel_to_copy_uop), + (UPat(Ops.CALL, src=(UPat(Ops.PARAM, name="dst").index(UPat(Ops.RANGE, name="r")+UPat.cvar("offset")) + .store(UPat(Ops.PARAM, name="src").index(UPat(Ops.RANGE, name="r"))).end(UPat(Ops.RANGE, name="r")).sink(),), + name="call", allow_any_len=True), copy_kernel_to_copy_uop), + (UPat(Ops.CALL, src=(UPat(Ops.PARAM, name="dst").index(UPat.cvar("offset")) + .store(UPat(Ops.PARAM, name="src").index(UPat(Ops.CONST, arg=0))).sink(),), + name="call", allow_any_len=True), copy_kernel_to_copy_uop), # if it wasn't copy, it currently can't be cross device (UPat(Ops.CALL, src=(UPat(Ops.SINK, name="ast"),), allow_any_len=True), assert_all_same_devices), diff --git a/tinygrad/schedule/prepare.py b/tinygrad/schedule/prepare.py index 3b14ef472ebd7..058fed74ae767 100644 --- a/tinygrad/schedule/prepare.py +++ b/tinygrad/schedule/prepare.py @@ -143,6 +143,28 @@ def copy_to_anon_store(x:UOp, copy:UOp): buf = UOp.new_buffer(copy.device, prod(x.max_shape), copy.dtype).reshape(x.max_shape) return buf.after(buf.store(x)).shrink_to(copy.shape) +def contiguous_buffer_view(x:UOp) -> tuple[UOp, int]|None: + if not isinstance(x.device, str) or (view:=x.contiguous_view()) is None: return None + base, offset = view + if (not base.has_buffer_identity(after_ok=True) or x.dtype != base.dtype or offset < 0 or + offset + prod(x.max_shape) > base.max_numel()): return None + return view + +def supports_sliced_copy_destination(x:UOp) -> bool: + return (isinstance(x.device, str) and + not x.device.startswith(("PYTHON", "NPY", "DISK", "CL", "WEBGPU", "NULL"))) + +def store_cross_device_copy(ctx:dict[UOp, int], dst:UOp, x:UOp, cpy:UOp): + if cpy.is_self_copy or dst.device != cpy.device or not supports_sliced_copy_destination(dst): return None + # COPY lowering encodes a concrete transfer length. Keep symbolic-sized stores on the ordinary COPY+STORE path. + if not all_int(dst.shape+x.shape): return None + if ctx.get(cpy) != 1 or contiguous_buffer_view(dst) is None or contiguous_buffer_view(x) is None: return None + return dst.store(x) + +pm_sliced_copy_store = PatternMatcher([ + (UPat(Ops.STORE, src=(UPat.var("dst"), UPat(Ops.COPY, src=(UPat.var("x"),), name="cpy"))), store_cross_device_copy), +]) + def materialize_cross_device_src(dest:UOp, src:UOp): # cross-device copies must read a whole buffer (SDMA can't do offset copies) if src.device is None or dest.device == src.device or src.has_buffer_identity(after_ok=True): return None @@ -219,5 +241,10 @@ def prepare_rangeify(sink:UOp) -> UOp: # prepare for rangeify tsink = graph_rewrite(sink, multi_pm, name="multi_pm") if OPENPILOT_HACKS: tsink = graph_rewrite(tsink, pm_fold_moved_after, ctx={}, name="fold moved afters") + uses:dict[UOp, int] = {} + for u in tsink.toposort(enter_calls=False): + for s in u.src: + if s.op is Ops.COPY: uses[s] = uses.get(s, 0) + 1 + tsink = graph_rewrite(tsink, pm_sliced_copy_store, ctx=uses, bottom_up=True, name="fold sliced copy store") tsink = graph_rewrite(tsink, pm_mops+earliest_rewrites, bottom_up=True, name="earliest rewrites") return tsink