Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 35 additions & 1 deletion test/backend/test_graph.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,11 @@ def run_schedule(calls:list[UOp]):
def zero_bufs(bufs):
for b in bufs: b.copy_from(Buffer("PYTHON", b.size, b.dtype, opaque=memoryview(bytearray(b.nbytes))))

def fill_int_buf(buf:Buffer, values:np.ndarray):
values = np.asarray(values, dtype=np.int32)
assert values.size == buf.size
buf.copy_from(Buffer("PYTHON", buf.size, buf.dtype, opaque=memoryview(bytearray(values.tobytes()))))

@unittest.skipUnless(Device[Device.DEFAULT].graph is not None, "graph support required")
class TestGraph(unittest.TestCase):
def skip_if_no_offset(self):
Expand All @@ -66,6 +71,35 @@ def skip_if_not_multigraph(self):
if not issubclass(graph, MultiGraphRunner): self.skipTest("graph is not supported (not MultiGraphRunner)")
if not hasattr(d.allocator, '_transfer') or not d.allocator.supports_transfer: self.skipTest("device is not supported (no transfers)")

@needs_second_gpu
def test_copy_graph_rebinds_nonzero_views(self):
self.skip_if_not_multigraph()
self.skip_if_no_offset()
d0, d1, base_size, copy_size = Device.DEFAULT, f"{Device.DEFAULT}:1", 16, 4
dstp, srcp = UOp.param(0, dtypes.int, base_size, d0), UOp.param(1, dtypes.int, base_size, d1)
copy = UOp(Ops.COPY, src=(UOp.param(1, dtypes.int, copy_size, d1),), arg=d0)
call = copy.call(dstp.shrink(((3, 3+copy_size),)), srcp.shrink(((5, 5+copy_size),)))
graph_call = UOp(Ops.CUSTOM_FUNCTION, src=(UOp(Ops.LINEAR, src=(call,)),), arg="graph")

initial_dst, initial_src = make_buffer(d0, base_size), make_buffer(d1, base_size)
initial = np.full(base_size, -1, dtype=np.int32)
fill_int_buf(initial_dst, initial)
fill_int_buf(initial_src, np.arange(base_size, dtype=np.int32))
graph = Device[d0].graph(graph_call, (UOp.from_buffer(initial_dst), UOp.from_buffer(initial_src)))

dst, src = make_buffer(d0, base_size), make_buffer(d1, base_size)
dstv, srcv = np.full(base_size, -2, dtype=np.int32), np.arange(base_size, dtype=np.int32) + 100
fill_int_buf(dst, dstv)
fill_int_buf(src, srcv)
graph((UOp.from_buffer(dst), UOp.from_buffer(src)), {})
Device[d0].synchronize()
Device[d1].synchronize()

expected = dstv.copy()
expected[3:3+copy_size] = srcv[5:5+copy_size]
np.testing.assert_equal(expected, np.frombuffer(dst.as_memoryview(), np.int32))
np.testing.assert_equal(initial, np.frombuffer(initial_dst.as_memoryview(), np.int32))

def test_order_2_writes_to_same_buf(self):
d0 = Device.DEFAULT
b = [make_buffer(d0, fill=True) for _ in range(5)]
Expand Down Expand Up @@ -315,4 +349,4 @@ def test_middle_write_splits_write_dep(self):
for buf in outs: np.testing.assert_equal(expected[buf], np.frombuffer(buf.as_memoryview(), np.int32))

if __name__ == '__main__':
unittest.main()
unittest.main()
28 changes: 27 additions & 1 deletion test/unit/test_jit.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,23 @@ def _simple_test(add, extract=lambda x: x, N=10):
assert_jit_cache_len(add, 1)

class TestJit(unittest.TestCase):
def test_graph_rebinds_copy_views(self):
from tinygrad.engine.jit import GraphRunner
from tinygrad.uop.ops import Ops
dst, src = UOp.param(0, dtypes.int32, 6, "CPU"), UOp.param(1, dtypes.int32, 5, "CPU:1")
copy = UOp(Ops.COPY, src=(UOp.param(1, dtypes.int32, 2, "CPU:1"),), arg="CPU")
call = copy.call(dst.shrink(((2, 4),)), src.shrink(((1, 3),)))
d0, s0 = Tensor.zeros(6, dtype=dtypes.int32, device="CPU").realize(), Tensor.zeros(5, dtype=dtypes.int32, device="CPU:1").realize()
graph = GraphRunner(UOp(Ops.CALL, src=(UOp(Ops.LINEAR, src=(call,)),)), (d0.uop, s0.uop))
d1 = Tensor.zeros(6, dtype=dtypes.int32, device="CPU").realize()
s1 = Tensor([10, 20, 30, 40, 50], device="CPU:1").realize()
replacements = dict(graph.updated_buffers(0, (d1.uop, s1.uop)))
self.assertIs(replacements[0].base, d1.uop.buffer.base)
self.assertIs(replacements[1].base, s1.uop.buffer.base)
self.assertEqual((replacements[0].offset, replacements[1].offset), (8, 4))
replacements[0].copy_from(replacements[1])
self.assertListEqual(d1.tolist(), [0, 0, 20, 30, 0, 0])

def test_jitbeam_triggers_beam(self):
from unittest.mock import patch
from tinygrad.helpers import getenv as _getenv
Expand Down Expand Up @@ -409,6 +426,15 @@ def w2(x) -> Tensor: return (weights*2).contiguous() + x
np.testing.assert_allclose(out.tolist(), [x*2+y for x,y in zip(weights.tolist(), a.tolist())])
assert_jit_cache_len(w2_prune, 1)

def test_prune_slice_assign_from_fixed_source(self):
src = Tensor([7, 8], device="CPU:1").realize()
@TinyJit(prune=True)
def write(dst): dst[1:3].assign(src.to(dst.device)).realize()
for fill in range(4):
dst = Tensor.full(4, fill, dtype=dtypes.int32, device="CPU").realize()
write(dst)
self.assertListEqual(dst.tolist(), [fill, 7, 8, fill])


class TestJitInsideJit(unittest.TestCase):
def test_jit_jit_error(self):
Expand Down Expand Up @@ -439,4 +465,4 @@ def f(): return Tensor.randint(20, high=5)
self.assertListEqual(t0, t1, msg=f"mismatch at list {i}")

if __name__ == '__main__':
unittest.main()
unittest.main()
69 changes: 65 additions & 4 deletions test/unit/test_setitem_schedule.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import unittest
from tinygrad import Tensor, Device, dtypes, GlobalCounters
from tinygrad import Tensor, Device, TinyJit, Variable, dtypes, GlobalCounters
from tinygrad.engine.realize import run_linear
from tinygrad.uop.ops import Ops
from test.helpers import assert_kernel_count

class TestSetitemInto(unittest.TestCase):
Expand Down Expand Up @@ -141,14 +143,73 @@ def test_setitem_full(self):

@unittest.skipUnless(Device.DEFAULT != "CPU", "source must be on another device")
def test_setitem_slice_assign_from_other_device(self):
# NOTE: this is 2 kernels, the cross-device copy should fuse with the assign into one
a = Tensor.ones(20, device="CPU")
b = Tensor.arange(20).float().clone()
Tensor.realize(a, b)
GlobalCounters.reset()
a[10:12].assign(b[13:15].to(a.device)).realize()
assert_kernel_count(2)
assert_kernel_count(2 if Device.DEFAULT.startswith(("PYTHON", "NPY", "DISK", "CL", "WEBGPU", "NULL")) else 1)
self.assertListEqual(a.tolist(), [1.0]*10 + [13.0, 14.0] + [1.0]*8)

def test_cross_device_slice_copy_targets_destination_view(self):
dst = Tensor(list(range(8)), dtype=dtypes.int32, device="CPU:1").realize()
src = Tensor(list(range(10, 18)), dtype=dtypes.int32, device="CPU").realize()
linear = dst[6:8].assign(src[3:5].to(dst.device)).schedule_linear()
self.assertEqual(len(linear.src), 1)
call = linear.src[0]
self.assertIs(call.src[0].op, Ops.COPY)
self.assertEqual((call.src[1].contiguous_view_offset(), call.src[2].contiguous_view_offset()), (6, 3))
self.assertEqual(call.src[1].max_numel(), 2)
run_linear(linear)
self.assertListEqual(dst.tolist(), [0, 1, 2, 3, 4, 5, 13, 14])

def test_cross_device_contiguous_reshape_uses_single_copy(self):
dst = Tensor(list(range(30)), device="CPU").realize()
src = Tensor(list(range(100, 110)), device="CPU:1").reshape(2, 5).realize()
linear = dst.reshape(6, 5)[2:4, :].assign(src.to(dst.device)).schedule_linear()
self.assertEqual([x.src[0].op for x in linear.src], [Ops.COPY])
run_linear(linear)
self.assertListEqual(dst.tolist(), list(range(10)) + list(range(100, 110)) + list(range(20, 30)))

def test_cross_device_noncontiguous_destination_falls_back(self):
dst = Tensor(list(range(12)), device="CPU").realize()
src = Tensor(list(range(100, 104)), device="CPU:1").realize()
linear = dst[1:9:2].assign(src.to(dst.device)).schedule_linear()
self.assertGreaterEqual(len(linear.src), 2)
run_linear(linear)
self.assertListEqual(dst.tolist(), [0, 100, 2, 101, 4, 102, 6, 103, 8, 9, 10, 11])

def test_cross_device_noncontiguous_source_falls_back(self):
dst = Tensor.zeros(2, dtype=dtypes.int32, device="CPU").realize()
src = Tensor(list(range(8)), dtype=dtypes.int32, device="CPU:1").reshape(2, 4).T[1]
linear = dst.assign(src.to(dst.device)).schedule_linear()
self.assertGreaterEqual(len(linear.src), 2)
run_linear(linear)
self.assertListEqual(dst.tolist(), [1, 5])

def test_cross_device_shared_copy_falls_back(self):
dst = Tensor.zeros(6, dtype=dtypes.int32, device="CPU").realize()
rhs = Tensor([7, 8], device="CPU:1").realize().to(dst.device)
dst[0:2].assign(rhs)
linear = dst[4:6].assign(rhs).schedule_linear()
self.assertGreaterEqual(len(linear.src), 2)
run_linear(linear)
self.assertListEqual(dst.tolist(), [7, 8, 0, 0, 7, 8])
self.assertListEqual(rhs.tolist(), [7, 8])

def test_symbolic_length_cross_device_slice_does_not_overcopy(self):
@TinyJit
def write(dst:Tensor, src:Tensor, v):
dst.shrink(((7, 7+v),)).assign(src.shrink(((9, 9+v),)).to(dst.device)).realize()
for i, value in enumerate((1, 4, 2)):
before = [-100-i]*20
values = list(range(100*i, 100*i+24))
dst = Tensor(before, dtype=dtypes.int32, device="CPU").realize()
src = Tensor(values, dtype=dtypes.int32, device="CPU:1").realize()
write(dst, src, Variable("slice_len", 1, 4).bind(value))
expected = before[:]
expected[7:7+value] = values[9:9+value]
self.assertListEqual(dst.tolist(), expected)

if __name__ == '__main__':
unittest.main()
unittest.main()
18 changes: 12 additions & 6 deletions tinygrad/engine/jit.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
from tinygrad.uop.ops import UOp, PatternMatcher, Variable, sym_infer, Ops, rewrite_group, graph_rewrite
from tinygrad.renderer import Estimates
from tinygrad.engine.realize import capturing, compile_linear, link_linear, run_linear, graph_cache, estimate_uop, get_runtime
from tinygrad.engine.realize import unwrap_multi, resolve_params, get_call_arg_uops, get_call_written_bufs
from tinygrad.engine.realize import unwrap_multi, resolve_params, get_call_arg_uops, get_call_written_bufs, _resolve
from tinygrad.schedule.memory import memory_plan_rewrite, _collect_bufs
from tinygrad.nn.state import get_parameters
from tinygrad.uop.movement import mop_cleanup
Expand All @@ -16,7 +16,7 @@
def prune_linear(linear:UOp, needed:set[UOp]) -> tuple[UOp, UOp]:
kept, onetime = [], []
for si in linear.src:
si_bufs = {b for src in si.src[1:] for b in _collect_bufs(src)}
si_bufs = {b for src in si.src[1:] for b in _collect_bufs(src.storage_base)}
if not si_bufs.isdisjoint(needed):
kept.append(si)
needed |= si_bufs
Expand Down Expand Up @@ -91,13 +91,14 @@ def __init__(self, linear:UOp, input_uops:tuple[UOp, ...]=()):
self.linear = linear.src[0]
self.calls: list[tuple[int, UOp, list[Buffer], dict[str, int]]] = []
self.runtimes: list[Any|None] = []
self.uop_replace: list[list[tuple[int, int]]] = []
self.uop_replace: list[list[tuple[int, UOp]]] = []
for call in self.linear.src:
replace = [(p, b.arg.slot) for p, b in enumerate(get_call_arg_uops(call)) if b.op is Ops.PARAM]
buffer_replace = [(p, b) for p, b in enumerate(get_call_arg_uops(call))
if b.op is Ops.PARAM or (call.src[0].op is Ops.COPY and _resolve(b, input_uops) is not b)]
for dev_idx, (bufs, device_vars) in enumerate(unwrap_multi(call, resolve_params(call, input_uops))):
self.calls.append((dev_idx, call.src[0], [b.ensure_allocated() for b in bufs], device_vars))
self.runtimes.append(get_runtime(bufs[0].device, call.src[0]) if call.src[0].op is Ops.PROGRAM else None)
self.uop_replace.append(replace)
self.uop_replace.append(buffer_replace)

self.var_vals_replace:dict[int, list[tuple[int, int]]] = {}
self.launch_dims_replace:dict[int, tuple[int|None, int|None]] = {}
Expand Down Expand Up @@ -128,6 +129,11 @@ def find_symbolic_dim(dim:tuple[int,int,int]): return self.symbolic_dims.index(t

def __call__(self, input_uops:tuple[UOp, ...], var_vals:dict[str, int], wait=False) -> float|None: raise NotImplementedError("override this")

def updated_buffers(self, j:int, input_uops:tuple[UOp, ...]):
for pos, u in self.uop_replace[j]:
buf = _resolve(u, input_uops).buffer
yield pos, (buf.bufs[self.calls[j][0]] if isinstance(buf, MultiBuffer) else buf).ensure_allocated()

def updated_vars(self, var_vals: dict[str, int]):
vals = [var_vals[v] for v in self.vars]
for j, vidxs in self.var_vals_replace.items():
Expand Down Expand Up @@ -291,4 +297,4 @@ def _buf_or_none(u:UOp) -> Buffer|MultiBuffer|None: return u.arg.buffer if u.op
def TinyJit(fxn:Callable[..., ReturnType], *, prune:bool=False) -> _TinyJit[ReturnType]: ...
@overload
def TinyJit(fxn:None=None, *, prune:bool=False) -> Callable[[Callable[..., ReturnType]], _TinyJit[ReturnType]]: ...
def TinyJit(fxn=None, **kwargs): return (lambda f: _TinyJit(f, **kwargs)) if fxn is None else _TinyJit(fxn, **kwargs)
def TinyJit(fxn=None, **kwargs): return (lambda f: _TinyJit(f, **kwargs)) if fxn is None else _TinyJit(fxn, **kwargs)
7 changes: 3 additions & 4 deletions tinygrad/runtime/graph/cuda.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
from typing import Any, cast
import tinygrad.runtime.autogen.cuda as cuda
from tinygrad.runtime.support.c import init_c_var
from tinygrad.device import Device, MultiBuffer
from tinygrad.device import Device
from tinygrad.uop.ops import UOp, Ops
from tinygrad.runtime.ops_cuda import CUDADevice, check, encode_args, cu_time_execution
from tinygrad.engine.jit import MultiGraphRunner
Expand Down Expand Up @@ -47,9 +47,8 @@ def new_node(self, bufs, write):
def __call__(self, input_uops:tuple[UOp, ...], var_vals:dict[str, int], wait=False):
# Update buffers in the c_args struct.
for j in self.updatable:
(_, params, c_args, is_copy), dev_idx = self.nodes[j], self.calls[j][0]
for pos, iidx in self.uop_replace[j]:
buf = b.bufs[dev_idx] if isinstance(b:=input_uops[iidx].buffer, MultiBuffer) else b
_, params, c_args, is_copy = self.nodes[j]
for pos, buf in self.updated_buffers(j, input_uops):
if not is_copy: setattr(c_args, f'f{pos}', buf._buf)
else: setattr(params, 'srcDevice' if pos == 1 else 'dstDevice', buf._buf)

Expand Down
5 changes: 2 additions & 3 deletions tinygrad/runtime/graph/metal.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
from typing import Any, cast
import ctypes, decimal, struct
from tinygrad.helpers import dedup, getenv, unwrap, PROFILE
from tinygrad.device import Buffer, Device, ProfileGraphEntry, ProfileGraphEvent
from tinygrad.device import Device, ProfileGraphEntry, ProfileGraphEvent
from tinygrad.uop.ops import UOp, Ops
from tinygrad.engine.jit import GraphRunner, GraphException
from tinygrad.runtime.ops_metal import MetalDevice, wait_check, to_ns_str
Expand Down Expand Up @@ -61,8 +61,7 @@ def __call__(self, input_uops:tuple[UOp, ...], var_vals:dict[str, int], wait=Fal
updated_bufs = []
for j in self.updatable:
computeCommand = self.icb.indirectComputeCommandAtIndex(j)
for pos, iidx in self.uop_replace[j]:
buf = cast(Buffer, input_uops[iidx].buffer)
for pos, buf in self.updated_buffers(j, input_uops):
computeCommand.setKernelBuffer_offset_atIndex(buf._buf.buf, buf._buf.offset, pos)
updated_bufs.append(buf._buf.buf)

Expand Down
28 changes: 25 additions & 3 deletions tinygrad/schedule/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ def create_schedule(sched_sink:UOp) -> UOp:

from tinygrad.schedule.memory import memory_plan_rewrite
from tinygrad.engine.realize import capturing, pm_flatten_linear
from tinygrad.schedule.prepare import prepare_rangeify
from tinygrad.schedule.prepare import prepare_rangeify, supports_sliced_copy_destination
from tinygrad.schedule.rangeify import get_kernel_graph
from tinygrad.helpers import CAPTURING
from tinygrad.uop.ops import PatternMatcher, UPat, ParamArg
Expand Down Expand Up @@ -152,9 +152,25 @@ def assert_all_same_devices(ast:UOp):
devices = dedup([x.device for x in ast.toposort() if x.op is Ops.PARAM and x.device is not None])
if len(devices) >= 2: raise RuntimeError(f"all buffers must be on the same device: {devices}")

def copy_kernel_to_copy_uop(call:UOp, dst:UOp, src:UOp, r:UOp|None=None):
def copy_kernel_to_copy_uop(call:UOp, dst:UOp, src:UOp, r:UOp|None=None, offset:UOp|None=None):
if dst.device == src.device and not (isinstance(dst.device, str) and dst.device.startswith("DISK")): return None
return call.replace(src=(UOp(Ops.COPY, src=(src,), arg=dst.device),) + call.src[1:])
# A runtime COPY has a fixed byte count. The RANGE must therefore cover the exact concrete source PARAM.
if r is not None and (len(r.src) != 1 or r.src[0].op is not Ops.CONST or r.src[0].val != src.arg.size): return None
copy = UOp(Ops.COPY, src=(src,), arg=dst.device)
if offset is None and (dst.arg.size == src.arg.size or not supports_sliced_copy_destination(dst)):
return call.replace(src=(copy,) + call.src[1:])
if offset is None: offset = UOp.const(0)
if (not supports_sliced_copy_destination(dst) or not isinstance(offset.val, int) or
dst.arg.slot != 0 or src.arg.slot != 1 or len(call.src) != 3 or
not isinstance(dst.device, str) or not isinstance(src.device, str) or
not isinstance(src.arg.size, int) or src.arg.size <= 0 or dst.dtype != src.dtype): return None
dest, source, size = call.src[1], call.src[2], src.arg.size
if (dest.dtype != dst.dtype or source.dtype != src.dtype or dest.max_numel() != dst.arg.size or source.max_numel() != size or
offset.val < 0 or offset.val + size > dest.max_numel()): return None
dest = dest.shrink(((offset.val, offset.val+size),))
if ((dview:=dest.contiguous_view()) is None or source.contiguous_view() is None or
dview[0].storage_base is not call.src[1].storage_base or dview[1] != offset.val): return None
return call.replace(src=(copy, dest, source))

def simplify_copy_kernel(call:UOp, ast:UOp, dst:UOp, src:UOp):
# NOTE: this is a codegen for SDMA devices
Expand All @@ -176,6 +192,12 @@ def simplify_copy_kernel(call:UOp, ast:UOp, dst:UOp, src:UOp):
(UPat(Ops.CALL, src=(UPat(Ops.PARAM, name="dst").index(UPat(Ops.RANGE, name="r"))
.store(UPat(Ops.PARAM, name="src").index(UPat(Ops.RANGE, name="r"))).end(UPat(Ops.RANGE, name="r")).sink(),),
name="call", allow_any_len=True), copy_kernel_to_copy_uop),
(UPat(Ops.CALL, src=(UPat(Ops.PARAM, name="dst").index(UPat(Ops.RANGE, name="r")+UPat.cvar("offset"))
.store(UPat(Ops.PARAM, name="src").index(UPat(Ops.RANGE, name="r"))).end(UPat(Ops.RANGE, name="r")).sink(),),
name="call", allow_any_len=True), copy_kernel_to_copy_uop),
(UPat(Ops.CALL, src=(UPat(Ops.PARAM, name="dst").index(UPat.cvar("offset"))
.store(UPat(Ops.PARAM, name="src").index(UPat(Ops.CONST, arg=0))).sink(),),
name="call", allow_any_len=True), copy_kernel_to_copy_uop),

# if it wasn't copy, it currently can't be cross device
(UPat(Ops.CALL, src=(UPat(Ops.SINK, name="ast"),), allow_any_len=True), assert_all_same_devices),
Expand Down
Loading
Loading