diff --git a/tools/qbs/README.md b/tools/qbs/README.md new file mode 100644 index 0000000000..d1424da09c --- /dev/null +++ b/tools/qbs/README.md @@ -0,0 +1,74 @@ +# qbs + +Quick, broad, shallow tests for a connected board. Each test is small and +runs on every target. The aim is to find obvious problems on many boards +quickly, not to test any one feature in depth. + + go run ./tools/qbs -target + +The tool flashes one program per class with `tinygo flash`, collects results +over the board's serial port and checks them. The tool builds with the +standard Go compiler. The programs it flashes build with TinyGo. + +Classes: + +- timing: sleep accuracy, a monotonic clock, loop speed, and the device clock + against the host clock +- compute: the same computations (integers, bits, sorting, maps, hashes, + encodings, strings, float32 and float64, math, strconv, fmt) run on the + device and on the host and the results are compared +- gc: large blocks, linked lists, slice and string growth, map churn, + fragmentation and collection cycles, sized from the free heap. The map and + fragmentation tests are not built for AVR. A conservative collector can keep + a whole linked structure alive through one stale pointer, so these tests + fail only when memory use keeps growing over repeated runs. +- serial: the default serial port. Bytes sent to the board come back + unchanged (echo), overflowing the RX buffer keeps the first bytes and + receiving still works (burst), 2000 numbered lines arrive in order (seq), all + 256 byte values arrive intact (bytes), and output and input work together + with goroutines and allocation (conc, not built on targets without a + scheduler) + +Use `-v` to see the values each test sent, `-port` if the board is not found, +`-tinygo` for a different tinygo binary and `-path-prepend` to put a flashing +tool first on PATH. Names after the flags select classes. + +Use `-emulator` to run the programs in the emulator of the target instead of +flashing a board, for example `go run ./tools/qbs -target wasip1 -emulator`. +The runner then starts `tinygo run` for each class and talks to the program +through its standard input and output. The emulator, such as wasmtime or QEMU, +must be installed. + +Some emulated targets cannot receive input, for example riscv-qemu and +cortex-m-qemu. Add `-noinput` for them. The programs then start without the +host and the serial class is skipped. + +simavr holds back its output when it is not a terminal and wraps each line in +colour codes, so add `-pty`. The tasks scheduler of the simavr target breaks an +explicit `runtime.GC()`, so build without a scheduler: + + go run ./tools/qbs -target simavr -emulator -noinput -pty -scheduler none + +`-gc` builds the programs with another garbage collector, for example +`-gc precise`. Use it for simavr. The default conservative collector takes +many values on a 16 bit AVR for addresses of the heap, and the gc tests then run +out of memory. `-class-timeout` limits how long one class may take, 10 minutes +by default. A program that prints garbage after it runs out of memory would +otherwise never end. + +A class is skipped when its program does not fit on the target. A test is +skipped when the target cannot run it, for example when the garbage collector +does not reclaim memory. + +## Adding a test + +- A computation for the compute class goes in `cases`. It must give the same + result on the host and the device. Use `ULP` for float results that may + differ in the last bits. +- Any other test is a `dev.Test` in the program of its class. Use `t.Fail`, + `t.Skip`, `t.Val` and `t.Raw`. A test that needs the host to judge the + result sends values, ticks or raw bytes and gets a check in the host code. + A test that needs the host to send data gets a driver, which runs on the + host when the test begins (see `serial.go`). +- A new class is a new directory with a program that calls `dev.Main`, and an + entry in the class list in `main.go`. diff --git a/tools/qbs/cases/cases.go b/tools/qbs/cases/cases.go new file mode 100644 index 0000000000..dca1c36435 --- /dev/null +++ b/tools/qbs/cases/cases.go @@ -0,0 +1,68 @@ +// Package cases has deterministic computations that run on the device and on +// the host. The host compares the two results. +package cases + +import "strconv" + +// Case is one computation. Fn returns a short string that describes the result. +type Case struct { + Name string + Fn func() string + // ULP is the allowed difference in units in the last place when Fn + // returns space separated hex float bits. Zero means an exact match. + ULP uint64 + // Heap is the number of bytes of heap the case needs. The device skips + // the case when less is free. + Heap int +} + +var all []Case + +func register(name string, ulp uint64, fn func() string) { + all = append(all, Case{Name: name, Fn: fn, ULP: ulp}) +} + +// needsHeap records how much heap the case registered last needs. +func needsHeap(bytes int) { + all[len(all)-1].Heap = bytes +} + +// All returns the cases that are built for this target. +func All() []Case { + return all +} + +// rng is a xorshift32 generator. +type rng uint32 + +func (r *rng) next() uint32 { + x := uint32(*r) + x ^= x << 13 + x ^= x >> 17 + x ^= x << 5 + *r = rng(x) + return x +} + +func randBytes(n int, seed uint32) []byte { + r := rng(seed) + b := make([]byte, n) + for i := range b { + b[i] = byte(r.next() >> 11) + } + return b +} + +func hex64(v uint64) string { + return strconv.FormatUint(v, 16) +} + +// fnv64 hashes s so that long results stay short. +func fnv64(s string) uint64 { + h := uint64(14695981039346656037) + for i := 0; i < len(s); i++ { + h ^= uint64(s[i]) + h *= 1099511628211 + } + return h +} diff --git a/tools/qbs/cases/float.go b/tools/qbs/cases/float.go new file mode 100644 index 0000000000..72fdca0184 --- /dev/null +++ b/tools/qbs/cases/float.go @@ -0,0 +1,86 @@ +package cases + +import ( + "math" + "strconv" + "strings" +) + +// The explicit conversions stop the host compiler from fusing a multiply and +// an add, which the soft float code on the device does not do. + +func init() { + register("float64-basic", 0, func() string { + r := rng(20) + var s float64 + acc := uint64(0) + for i := 0; i < 500; i++ { + x := float64(int32(r.next())) / 65536.0 + y := float64(int32(r.next()))/1024.0 + 0.5 + s += float64(x * y) + s -= float64(x / (y + 1e6)) + s = float64(s*0.999) + math.Sqrt(math.Abs(x)) + acc = acc*31 + math.Float64bits(s) + } + return hex64(acc) + " " + hex64(math.Float64bits(s)) + }) + + register("float32-basic", 0, func() string { + r := rng(21) + var s float32 + acc := uint64(0) + for i := 0; i < 500; i++ { + x := float32(int32(r.next())) / 65536.0 + y := float32(int32(r.next()))/1024.0 + 0.5 + s += float32(x * y) + s -= float32(x / (y + 1e6)) + s = float32(s*0.999) + float32(math.Sqrt(math.Abs(float64(x)))) + acc = acc*31 + uint64(math.Float32bits(s)) + } + return hex64(acc) + " " + hex64(uint64(math.Float32bits(s))) + }) + + register("math-funcs", 8, func() string { + var out []string + xs := []float64{0.1, 0.5, 1, 1.5, 2.5, 3.14159, 7, 10.25, 100.5, 0.001} + for _, x := range xs { + for _, v := range []float64{ + math.Sin(x), math.Cos(x), math.Tan(x), math.Exp(x / 4), math.Log(x), + math.Pow(x, 1.7), math.Atan2(x, 3), math.Sinh(x / 5), math.Cbrt(x), + } { + out = append(out, hex64(math.Float64bits(v))) + } + } + return strings.Join(out, " ") + }) + + register("strconv", 0, func() string { + r := rng(22) + var sb strings.Builder + for i := 0; i < 60; i++ { + f := math.Float64frombits(uint64(r.next())<<32|uint64(r.next())) / 1e30 + if math.IsNaN(f) || math.IsInf(f, 0) { + f = float64(i) + } + sb.WriteString(strconv.FormatFloat(f, 'g', -1, 64)) + sb.WriteString(strconv.FormatFloat(f, 'e', 6, 64)) + sb.WriteString(strconv.FormatFloat(f, 'f', 3, 32)) + back, err := strconv.ParseFloat(strconv.FormatFloat(f, 'g', -1, 64), 64) + if err != nil || back != f { + sb.WriteString("roundtrip-failed") + } + } + for base := 2; base <= 36; base++ { + v := int64(int32(r.next())) + s := strconv.FormatInt(v, base) + back, err := strconv.ParseInt(s, base, 64) + if err != nil || back != v { + sb.WriteString("int-roundtrip-failed") + } + sb.WriteString(s) + } + s := sb.String() + return strconv.Itoa(len(s)) + " " + hex64(fnv64(s)) + }) + needsHeap(8 << 10) +} diff --git a/tools/qbs/cases/fmt.go b/tools/qbs/cases/fmt.go new file mode 100644 index 0000000000..ef28f3d760 --- /dev/null +++ b/tools/qbs/cases/fmt.go @@ -0,0 +1,18 @@ +//go:build !avr + +package cases + +import "fmt" + +func init() { + register("fmt", 0, func() string { + r := rng(30) + s := "" + for i := 0; i < 20; i++ { + v := int32(r.next()) + s += fmt.Sprintf("%d %x %08.3f %q %v %5.1f|", v, uint32(v), float64(v)/1000, fmt.Sprint(i), []int{i, i + 1}, float32(i)/3) + } + return fmt.Sprintf("%d %x", len(s), fnv64(s)) + }) + needsHeap(8 << 10) +} diff --git a/tools/qbs/cases/hash.go b/tools/qbs/cases/hash.go new file mode 100644 index 0000000000..dc4300d5a2 --- /dev/null +++ b/tools/qbs/cases/hash.go @@ -0,0 +1,80 @@ +package cases + +import ( + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "encoding/base64" + "encoding/binary" + "encoding/hex" + "hash/adler32" + "hash/crc32" + "hash/fnv" + "strings" + "unicode/utf8" +) + +func init() { + register("crc32", 0, func() string { + data := randBytes(1500, 10) + castagnoli := crc32.MakeTable(crc32.Castagnoli) + sum := crc32.Update(0, castagnoli, data[:700]) + sum = crc32.Update(sum, castagnoli, data[700:]) + return hex64(uint64(crc32.ChecksumIEEE(data))) + " " + hex64(uint64(sum)) + }) + // Two slicing tables of 8KB each. + needsHeap(20 << 10) + + register("adler32-fnv", 0, func() string { + data := randBytes(1000, 11) + h := fnv.New64a() + h.Write(data) + h32 := fnv.New32() + h32.Write(data) + return hex64(uint64(adler32.Checksum(data))) + " " + hex64(h.Sum64()) + " " + hex64(uint64(h32.Sum32())) + }) + + register("sha256", 0, func() string { + sum := sha256.Sum256(randBytes(1000, 12)) + return hex.EncodeToString(sum[:]) + }) + needsHeap(3 << 10) + + register("sha1-md5", 0, func() string { + data := randBytes(777, 13) + s1 := sha1.Sum(data) + m5 := md5.Sum(data) + return hex.EncodeToString(s1[:]) + " " + hex.EncodeToString(m5[:]) + }) + + register("encoding", 0, func() string { + data := randBytes(300, 14) + b64 := base64.StdEncoding.EncodeToString(data) + back, err := base64.StdEncoding.DecodeString(b64) + if err != nil || string(back) != string(data) { + return "base64 round trip failed" + } + raw := base64.RawURLEncoding.EncodeToString(data[:101]) + buf := make([]byte, 16) + binary.LittleEndian.PutUint64(buf, binary.BigEndian.Uint64(data)) + binary.BigEndian.PutUint32(buf[8:], binary.LittleEndian.Uint32(data[8:])) + return hex64(fnv64(b64)) + " " + hex64(fnv64(raw)) + " " + hex.EncodeToString(buf) + }) + needsHeap(4 << 10) + + register("strings", 0, func() string { + var sb strings.Builder + r := rng(15) + for i := 0; i < 40; i++ { + sb.WriteRune(rune(0x20 + r.next()%0x2000)) + sb.WriteString(" ab,cd;") + } + s := sb.String() + parts := strings.FieldsFunc(s, func(r rune) bool { return r == ',' || r == ';' }) + joined := strings.Join(parts, "|") + up := strings.ToUpper(strings.Repeat(joined[:20], 3)) + rep := strings.ReplaceAll(s, "ab", "xyz") + return hex64(fnv64(joined)) + " " + hex64(fnv64(up)) + " " + hex64(fnv64(rep)) + + " " + hex64(uint64(utf8.RuneCountInString(s))) + " " + hex64(uint64(strings.Index(s, "cd;"))) + }) +} diff --git a/tools/qbs/cases/integer.go b/tools/qbs/cases/integer.go new file mode 100644 index 0000000000..356fc32e2c --- /dev/null +++ b/tools/qbs/cases/integer.go @@ -0,0 +1,96 @@ +package cases + +import ( + "math/bits" + "sort" + "strconv" +) + +func init() { + register("int-arith", 0, func() string { + r := rng(1) + acc := uint64(1469598103934665603) + for i := 0; i < 2000; i++ { + a := int64(int32(r.next())) * int64(int32(r.next())) + b := int64(r.next()%1000) + 1 + u := uint64(r.next())<<32 | uint64(r.next()) + v := uint64(r.next()) | 1 + acc = acc*1099511628211 ^ uint64(a) + acc ^= uint64(a / b) + acc ^= uint64(a % b) + acc ^= u / v + acc ^= u % v + acc += uint64(int32(r.next()) >> uint(i%31)) + acc ^= u<>uint(i%64) + } + return hex64(acc) + }) + + register("bits", 0, func() string { + r := rng(2) + var acc uint64 + for i := 0; i < 1000; i++ { + x := uint64(r.next())<<32 | uint64(r.next()) + y := uint32(r.next()) + hi, lo := bits.Mul64(x, uint64(y)|1) + sum, carry := bits.Add64(x, lo, uint64(i&1)) + acc += uint64(bits.LeadingZeros64(x)) + uint64(bits.TrailingZeros32(y)) + acc += uint64(bits.OnesCount64(x)) + uint64(bits.Len32(y)) + acc ^= bits.RotateLeft64(x, i%64) ^ uint64(bits.Reverse32(y)) ^ hi ^ sum ^ carry + } + return hex64(acc) + }) + + register("sort", 0, func() string { + r := rng(3) + ints := make([]int, 400) + for i := range ints { + ints[i] = int(int32(r.next())) + } + sort.Ints(ints) + var acc uint64 + for i, v := range ints { + acc = acc*31 + uint64(v)*uint64(i+1) + } + strs := make([]string, 60) + for i := range strs { + strs[i] = strconv.Itoa(int(r.next() % 100000)) + } + sort.Strings(strs) + for _, s := range strs { + acc = acc*131 + fnv64(s) + } + type pair struct{ k, v int } + pairs := make([]pair, 100) + for i := range pairs { + pairs[i] = pair{int(r.next() % 50), i} + } + sort.SliceStable(pairs, func(i, j int) bool { return pairs[i].k < pairs[j].k }) + for _, p := range pairs { + acc = acc*7 + uint64(p.k*1000+p.v) + } + return hex64(acc) + }) + needsHeap(8 << 10) + + register("map", 0, func() string { + r := rng(4) + m := make(map[uint32]uint32) + for i := 0; i < 300; i++ { + m[r.next()%500] = r.next() + } + for k := uint32(0); k < 500; k += 3 { + delete(m, k) + } + keys := make([]int, 0, len(m)) + for k := range m { + keys = append(keys, int(k)) + } + sort.Ints(keys) + var acc uint64 + for _, k := range keys { + acc = acc*1000003 ^ uint64(k)<<32 ^ uint64(m[uint32(k)]) + } + return strconv.Itoa(len(m)) + " " + hex64(acc) + }) +} diff --git a/tools/qbs/compute/main.go b/tools/qbs/compute/main.go new file mode 100644 index 0000000000..a64ffb25ea --- /dev/null +++ b/tools/qbs/compute/main.go @@ -0,0 +1,34 @@ +//go:build tinygo + +package main + +import ( + "runtime" + "strconv" + + "github.com/tinygo-org/tinygo/tools/qbs/cases" + "github.com/tinygo-org/tinygo/tools/qbs/dev" +) + +func main() { + var tests []dev.Test + for _, c := range cases.All() { + c := c + tests = append(tests, dev.Test{ + Name: c.Name, + Run: func(t *dev.T) { + if c.Heap > 0 { + runtime.GC() + var m runtime.MemStats + runtime.ReadMemStats(&m) + if int(m.HeapIdle) < c.Heap+c.Heap/2 { + t.Skip("needs " + strconv.Itoa(c.Heap) + " bytes of heap, " + strconv.Itoa(int(m.HeapIdle)) + " free") + return + } + } + t.Val("result", c.Fn()) + }, + }) + } + dev.Main("compute", tests) +} diff --git a/tools/qbs/dev/dev.go b/tools/qbs/dev/dev.go new file mode 100644 index 0000000000..95d724f6d6 --- /dev/null +++ b/tools/qbs/dev/dev.go @@ -0,0 +1,104 @@ +//go:build tinygo + +// Package dev is the device side of qbs. A program lists its tests and calls +// Main, which waits for the host, runs the tests and reports over serial. +package dev + +import "time" + +// Test is one named check. +type Test struct { + Name string + Run func(t *T) +} + +// T collects the outcome of one test. +type T struct { + failed bool + skipped bool + detail string +} + +// Fail marks the test as failed. Only the first message is kept. +func (t *T) Fail(msg string) { + if !t.failed { + t.failed = true + t.detail = msg + } +} + +// Skip marks the test as skipped. +func (t *T) Skip(msg string) { + t.skipped = true + t.detail = msg +} + +// Log sends a message to the host. +func (t *T) Log(msg string) { + println("@@LOG", msg) +} + +// Val sends a named value to the host. +func (t *T) Val(key, value string) { + println("@@VAL", key, value) +} + +// Raw sends bytes to the host unchanged. The host reads exactly len(data) +// bytes after the header line. +func (t *T) Raw(key string, data []byte) { + println("@@RAW", key, len(data)) + Serial.Write(data) + println() +} + +const hexDigits = "0123456789abcdef" + +// Hex returns the lowercase hex encoding of b. +func Hex(b []byte) string { + out := make([]byte, 0, len(b)*2) + for _, c := range b { + out = append(out, hexDigits[c>>4], hexDigits[c&15]) + } + return string(out) +} + +// ReadN reads up to n bytes from the serial port, giving up after timeout. +func ReadN(n int, timeout time.Duration) []byte { + var got []byte + deadline := time.Now().Add(timeout) + for len(got) < n && time.Now().Before(deadline) { + if Serial.Buffered() == 0 { + time.Sleep(time.Millisecond) + continue + } + b, _ := Serial.ReadByte() + got = append(got, b) + } + return got +} + +// Main runs the tests of one class and never returns. +func Main(class string, tests []Test) { + Serial.Configure(115200) + waitGo(class) + println("@@GO", class) + for i := range tests { + println("@@BEGIN", tests[i].Name) + t := &T{} + start := time.Now() + tests[i].Run(t) + us := time.Since(start).Microseconds() + status := "PASS" + switch { + case t.skipped: + status = "SKIP" + case t.failed: + status = "FAIL" + } + println("@@RESULT", tests[i].Name, status, us, t.detail) + } + for { + println("@@END", class) + time.Sleep(time.Second) + } +} diff --git a/tools/qbs/dev/handshake.go b/tools/qbs/dev/handshake.go new file mode 100644 index 0000000000..8c143e3e15 --- /dev/null +++ b/tools/qbs/dev/handshake.go @@ -0,0 +1,18 @@ +//go:build tinygo && !qbs_noinput + +package dev + +import "time" + +func waitGo(class string) { + for { + for Serial.Buffered() > 0 { + b, _ := Serial.ReadByte() + if b == 'G' { + return + } + } + println("@@READY qbs", class) + time.Sleep(300 * time.Millisecond) + } +} diff --git a/tools/qbs/dev/handshake_noinput.go b/tools/qbs/dev/handshake_noinput.go new file mode 100644 index 0000000000..f2705da077 --- /dev/null +++ b/tools/qbs/dev/handshake_noinput.go @@ -0,0 +1,6 @@ +//go:build tinygo && qbs_noinput + +package dev + +// waitGo does not wait. The target cannot receive input from the host. +func waitGo(class string) {} diff --git a/tools/qbs/dev/serial_machine.go b/tools/qbs/dev/serial_machine.go new file mode 100644 index 0000000000..41718f0df9 --- /dev/null +++ b/tools/qbs/dev/serial_machine.go @@ -0,0 +1,31 @@ +//go:build tinygo && baremetal && !qbs_noinput + +package dev + +import "machine" + +// Serial is the default serial port of the board. +var Serial serial + +type serial struct{} + +// Configure sets the baud rate. +func (serial) Configure(baud uint32) { + machine.Serial.Configure(machine.UARTConfig{BaudRate: baud}) +} + +func (serial) Buffered() int { + return machine.Serial.Buffered() +} + +func (serial) ReadByte() (byte, error) { + return machine.Serial.ReadByte() +} + +func (serial) Write(b []byte) (int, error) { + return machine.Serial.Write(b) +} + +func (serial) WriteByte(c byte) error { + return machine.Serial.WriteByte(c) +} diff --git a/tools/qbs/dev/serial_noinput.go b/tools/qbs/dev/serial_noinput.go new file mode 100644 index 0000000000..6e2c2ccbc9 --- /dev/null +++ b/tools/qbs/dev/serial_noinput.go @@ -0,0 +1,37 @@ +//go:build tinygo && qbs_noinput + +package dev + +// Serial has no input and writes through the runtime. Some targets, such as +// the QEMU ones, cannot receive input and have no machine package to use. +var Serial serial + +type serial struct{} + +func (serial) Configure(baud uint32) {} + +func (serial) Buffered() int { + return 0 +} + +func (serial) ReadByte() (byte, error) { + return 0, errNoInput +} + +func (serial) Write(b []byte) (int, error) { + print(string(b)) + return len(b), nil +} + +func (serial) WriteByte(c byte) error { + print(string([]byte{c})) + return nil +} + +type noInputError struct{} + +func (noInputError) Error() string { + return "no input on this target" +} + +var errNoInput error = noInputError{} diff --git a/tools/qbs/dev/serial_wasip1.go b/tools/qbs/dev/serial_wasip1.go new file mode 100644 index 0000000000..c8a1ebdc9f --- /dev/null +++ b/tools/qbs/dev/serial_wasip1.go @@ -0,0 +1,103 @@ +//go:build tinygo && wasip1 && !qbs_noinput + +package dev + +import ( + "os" + "unsafe" +) + +// Serial is the standard input and output of the program. The runner talks to +// the emulated program through them. +var Serial serial + +type serial struct{} + +// rx holds bytes read from standard input that have not been returned yet. +var rx []byte + +// Configure does nothing, there is no baud rate. +func (serial) Configure(baud uint32) {} + +// Buffered returns the number of bytes that can be read without blocking. +func (serial) Buffered() int { + if len(rx) == 0 && stdinReady() { + var buf [64]byte + n, _ := os.Stdin.Read(buf[:]) + rx = append(rx, buf[:n]...) + } + return len(rx) +} + +func (s serial) ReadByte() (byte, error) { + if s.Buffered() == 0 { + return 0, os.ErrDeadlineExceeded + } + b := rx[0] + rx = rx[1:] + return b, nil +} + +func (serial) Write(b []byte) (int, error) { + return os.Stdout.Write(b) +} + +func (s serial) WriteByte(c byte) error { + _, err := s.Write([]byte{c}) + return err +} + +// The layouts follow the WASI preview 1 subscription and event structs. +type subscription struct { + userdata uint64 + tag uint8 + _ [7]byte + fdOrClock uint32 + _ uint32 + timeout uint64 + precision uint64 + flags uint16 + _ [6]byte +} + +type event struct { + userdata uint64 + errno uint16 + typ uint8 + _ [5]byte + nbytes uint64 + flags uint16 + _ [6]byte +} + +const ( + eventClock = 0 + eventFDRead = 1 + clockMono = 1 +) + +//go:wasmimport wasi_snapshot_preview1 poll_oneoff +func pollOneoff(in unsafe.Pointer, out unsafe.Pointer, nsubscriptions uint32, nevents unsafe.Pointer) uint32 + +var ( + pollSubs [2]subscription + pollEvents [2]event +) + +// stdinReady reports whether reading standard input would not block. +func stdinReady() bool { + pollSubs[0] = subscription{tag: eventFDRead, fdOrClock: 0} + // With a zero timeout the clock is already due and poll_oneoff may return + // only that event, so give input a short time to be reported. + pollSubs[1] = subscription{tag: eventClock, fdOrClock: clockMono, timeout: 200_000} + var n uint32 + if pollOneoff(unsafe.Pointer(&pollSubs[0]), unsafe.Pointer(&pollEvents[0]), 2, unsafe.Pointer(&n)) != 0 { + return false + } + for i := uint32(0); i < n; i++ { + if pollEvents[i].typ == eventFDRead && pollEvents[i].errno == 0 { + return true + } + } + return false +} diff --git a/tools/qbs/emulate.go b/tools/qbs/emulate.go new file mode 100644 index 0000000000..22e600a76e --- /dev/null +++ b/tools/qbs/emulate.go @@ -0,0 +1,219 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "time" +) + +// conn is what the runner needs from a serial port or an emulated program. +type conn interface { + Read(p []byte) (int, error) + Write(p []byte) (int, error) + Close() error +} + +// pipePort talks to a program run by tinygo run. Its Read gives up after a +// short time without data, like a serial port with a read timeout. +type pipePort struct { + cmd *exec.Cmd + in io.WriteCloser + out chan []byte + pending []byte + closed bool + // cleanup removes what the program left behind. + cleanup func() +} + +// exited reports whether the program ended and all its output was read. +func (p *pipePort) exited() bool { + return p.closed && len(p.pending) == 0 +} + +func (p *pipePort) Read(b []byte) (int, error) { + if len(p.pending) == 0 { + select { + case data, ok := <-p.out: + if !ok { + p.closed = true + time.Sleep(50 * time.Millisecond) + return 0, nil + } + p.pending = data + case <-time.After(50 * time.Millisecond): + return 0, nil + } + } + n := copy(b, p.pending) + p.pending = p.pending[n:] + return n, nil +} + +func (p *pipePort) Write(b []byte) (int, error) { + return p.in.Write(b) +} + +func (p *pipePort) Close() error { + p.in.Close() + killGroup(p.cmd) + p.cmd.Wait() + if p.cleanup != nil { + p.cleanup() + } + return nil +} + +// buildFlags returns the flags that change how the programs are built. +func buildFlags(o options) []string { + var flags []string + if o.noInput { + flags = append(flags, "-tags", "qbs_noinput") + } + if o.scheduler != "" { + flags = append(flags, "-scheduler", o.scheduler) + } + if o.gc != "" { + flags = append(flags, "-gc", o.gc) + } + return flags +} + +// startEmulator runs the program of a class in the emulator of the target. +// Output and input go through the returned link. +func startEmulator(o options, dir string) (*link, error) { + if o.pty { + return startEmulatorPTY(o, dir) + } + args := append([]string{"run", "-target", o.target}, buildFlags(o)...) + cmd := exec.Command(o.tinygo, append(args, dir)...) + cmd.Dir = dir + return connect(o, cmd, nil) +} + +// startEmulatorPTY builds the program and starts the emulator itself under a +// pseudo terminal. tinygo run gives the emulator a pipe, and an emulator such +// as simavr then holds back its output until the program ends. +func startEmulatorPTY(o options, dir string) (*link, error) { + info, err := exec.Command(o.tinygo, "info", "-json", "-target", o.target).Output() + if err != nil { + return nil, fmt.Errorf("tinygo info failed: %v", err) + } + var spec struct { + Target struct { + Emulator string `json:"emulator"` + } `json:"target"` + } + if err := json.Unmarshal(info, &spec); err != nil || spec.Target.Emulator == "" { + return nil, fmt.Errorf("the target %s has no emulator", o.target) + } + tmp, err := os.MkdirTemp("", "qbs") + if err != nil { + return nil, err + } + prog := filepath.Join(tmp, "prog.elf") + build := append([]string{"build", "-target", o.target, "-o", prog}, buildFlags(o)...) + if out, err := exec.Command(o.tinygo, append(build, dir)...).CombinedOutput(); err != nil { + os.RemoveAll(tmp) + return nil, fmt.Errorf("build failed: %s", out) + } + words := strings.Fields(strings.ReplaceAll(spec.Target.Emulator, "{}", prog)) + var cmd *exec.Cmd + if runtime.GOOS == "darwin" { + cmd = exec.Command("script", append([]string{"-q", "/dev/null"}, words...)...) + } else { + cmd = exec.Command("script", "-qfec", shellJoin(words), "/dev/null") + } + cmd.Dir = dir + cleanup := func() { + // script starts the emulator in its own session, so stop it by the + // path of the program, which is unique to this run. + exec.Command("pkill", "-f", prog).Run() + os.RemoveAll(tmp) + } + l, err := connect(o, cmd, cleanup) + if err != nil { + cleanup() + } + return l, err +} + +// connect starts the command and connects its input and output to a link. +func connect(o options, cmd *exec.Cmd, cleanup func()) (*link, error) { + if o.pathPrepend != "" { + cmd.Env = append(os.Environ(), "PATH="+o.pathPrepend+string(os.PathListSeparator)+os.Getenv("PATH")) + } + setProcessGroup(cmd) + in, err := cmd.StdinPipe() + if err != nil { + return nil, err + } + r, w, err := os.Pipe() + if err != nil { + return nil, err + } + // Compiler messages and the program's own output arrive on one stream. + cmd.Stdout = w + cmd.Stderr = w + if err := cmd.Start(); err != nil { + return nil, err + } + w.Close() + out := make(chan []byte, 64) + go func() { + defer close(out) + for { + buf := make([]byte, 4096) + n, err := r.Read(buf) + if n > 0 { + out <- buf[:n] + } + if err != nil { + return + } + } + }() + return &link{port: &pipePort{cmd: cmd, in: in, out: out, cleanup: cleanup}}, nil +} + +// shellJoin quotes the arguments for a shell command line. +func shellJoin(args []string) string { + quoted := make([]string, len(args)) + for i, a := range args { + quoted[i] = "'" + strings.ReplaceAll(a, "'", `'\''`) + "'" + } + return strings.Join(quoted, " ") +} + +// emulatorFailure returns a short reason when the program could not run +// because the emulator is missing or the program does not fit on the target. +func emulatorFailure(log []byte) string { + text := string(log) + if i := strings.Index(text, "executable file not found"); i >= 0 { + return "emulator not installed: " + strings.TrimSpace(text[max(0, i-60):i+len("executable file not found")]) + } + return fitFailure(text) +} + +// fitFailure returns a reason, with the line of the error, when the build +// output says that the program does not fit on the target. The markers are +// kept narrow so that other link errors still fail. +func fitFailure(text string) string { + for _, line := range strings.Split(text, "\n") { + for _, marker := range []string{"overflowed", "will not fit", "is out of range"} { + if strings.Contains(line, marker) { + line = strings.TrimSpace(line) + if len(line) > 110 { + line = line[:107] + "..." + } + return "program does not fit on this target: " + line + } + } + } + return "" +} diff --git a/tools/qbs/emulate_other.go b/tools/qbs/emulate_other.go new file mode 100644 index 0000000000..31b0bcdffc --- /dev/null +++ b/tools/qbs/emulate_other.go @@ -0,0 +1,13 @@ +//go:build !unix + +package main + +import "os/exec" + +func setProcessGroup(cmd *exec.Cmd) {} + +func killGroup(cmd *exec.Cmd) { + if cmd.Process != nil { + cmd.Process.Kill() + } +} diff --git a/tools/qbs/emulate_unix.go b/tools/qbs/emulate_unix.go new file mode 100644 index 0000000000..53ef0344ea --- /dev/null +++ b/tools/qbs/emulate_unix.go @@ -0,0 +1,19 @@ +//go:build unix + +package main + +import ( + "os/exec" + "syscall" +) + +func setProcessGroup(cmd *exec.Cmd) { + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} +} + +// killGroup stops the program and the emulator it started. +func killGroup(cmd *exec.Cmd) { + if cmd.Process != nil { + syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + } +} diff --git a/tools/qbs/gc/heavy.go b/tools/qbs/gc/heavy.go new file mode 100644 index 0000000000..eb60ec74b7 --- /dev/null +++ b/tools/qbs/gc/heavy.go @@ -0,0 +1,109 @@ +//go:build tinygo && !avr + +package main + +import ( + "runtime" + + "github.com/tinygo-org/tinygo/tools/qbs/dev" +) + +func init() { + extraTests = append(extraTests, + dev.Test{Name: "map-churn", Run: mapChurn}, + dev.Test{Name: "fragmentation", Run: fragmentation}, + ) +} + +//go:noinline +func mapTest(n int) bool { + m := make(map[int]*[8]uint32) + for i := 0; i < n; i++ { + v := new([8]uint32) + v[0] = uint32(i) + v[7] = uint32(i) * 3 + m[i] = v + } + for i := 0; i < n; i += 2 { + delete(m, i) + } + for i := 0; i < n; i++ { + v, ok := m[i] + if i%2 == 0 { + if ok { + return false + } + } else if !ok || v[0] != uint32(i) || v[7] != uint32(i)*3 { + return false + } + } + for i := 0; i < n; i += 2 { + m[i] = new([8]uint32) + } + return len(m) == n +} + +func mapChurn(t *dev.T) { + b := budget(t, 6) + if b == 0 { + return + } + n := b / 100 + if n < 4 { + t.Skip("heap too small") + return + } + noGrowth(t, "map", b, func() bool { return mapTest(n) }) +} + +//go:noinline +func fragTest(count int) bool { + blocks := make([][]byte, count) + size := func(i, round int) int { return 16 + (i*37+round*11)%240 } + fill := func(i, round int) { + b := make([]byte, size(i, round)) + for j := range b { + b[j] = byte(i*5 + round + j) + } + blocks[i] = b + } + for i := range blocks { + fill(i, 0) + } + for i := 0; i < count; i += 2 { + blocks[i] = nil + } + runtime.GC() + for i := 0; i < count; i += 2 { + fill(i, 1) + } + for i := range blocks { + round := 0 + if i%2 == 0 { + round = 1 + } + b := blocks[i] + if len(b) != size(i, round) { + return false + } + for j := range b { + if b[j] != byte(i*5+round+j) { + return false + } + } + } + return true +} + +func fragmentation(t *dev.T) { + b := budget(t, 4) + if b == 0 { + return + } + count := b / (136 + 24 + 8) + if count < 8 { + t.Skip("heap too small") + return + } + noGrowth(t, "fragmentation blocks", b, func() bool { return fragTest(count) }) +} diff --git a/tools/qbs/gc/main.go b/tools/qbs/gc/main.go new file mode 100644 index 0000000000..15375e443e --- /dev/null +++ b/tools/qbs/gc/main.go @@ -0,0 +1,343 @@ +//go:build tinygo + +package main + +import ( + "runtime" + "strconv" + "strings" + "time" + + "github.com/tinygo-org/tinygo/tools/qbs/dev" +) + +var sink any + +// noReclaim is set when the collector does not give memory back. +var noReclaim bool + +// extraTests holds tests that need more flash or RAM than every target has. +var extraTests []dev.Test + +func main() { + // The tests that keep the most memory alive come last, and gc-cycles is the + // last of the tests that run everywhere. If the collector does not free + // memory, they use up the heap, and a program that runs out of memory may + // not report anything after that. + tests := []dev.Test{ + {Name: "reclaims", Run: reclaims}, + {Name: "large-block", Run: largeBlock}, + {Name: "strings", Run: stringsTest}, + {Name: "slice-grow", Run: sliceGrow}, + {Name: "linked-list", Run: linkedList}, + {Name: "gc-cycles", Run: gcCycles}, + } + dev.Main("gc", append(tests, extraTests...)) +} + +func stats() runtime.MemStats { + var m runtime.MemStats + runtime.ReadMemStats(&m) + return m +} + +// clobberStack overwrites the stack and registers so that stale pointers from +// earlier calls do not keep memory alive in a conservative collector. +// +//go:noinline +func clobberStack(n int) byte { + var buf [64]byte + for i := range buf { + buf[i] = byte(i + n) + } + if n > 0 { + return buf[n%64] + clobberStack(n-1) + } + return buf[0] +} + +// churn allocates a few small objects. A conservative collector can keep the +// most recent large allocation alive through a stale pointer until later +// allocations overwrite it. +// +//go:noinline +func churn() { + for i := 0; i < 8; i++ { + sink = make([]byte, 16+i*3) + } + sink = nil +} + +func collect() { + sink = clobberStack(4) + sink = nil + churn() + runtime.GC() +} + +// maxHeap limits the heap the tests use. A target with a very large heap, such +// as an emulator, would otherwise make them slow, and a conservative collector +// retains more of a large heap by mistaking data for pointers. +const maxHeap = 4 << 20 + +func freeHeap() int { + collect() + m := stats() + if m.HeapIdle > maxHeap { + return maxHeap + } + return int(m.HeapIdle) +} + +func inUse() int { + collect() + m := stats() + return int(m.HeapInuse) +} + +// budget returns the number of bytes a test may allocate, or 0 when the test +// should not run. +func budget(t *dev.T, divisor int) int { + if noReclaim { + t.Skip("the collector does not reclaim memory") + return 0 + } + b := freeHeap() / divisor + if b < 512 { + t.Skip("heap too small") + return 0 + } + return b +} + +// checkLeak fails the test when much more than the largest object is still in +// use. A conservative collector can keep the most recent large object alive +// through a stale pointer. +func checkLeak(t *dev.T, before, allocated, largest int) { + after := inUse() + t.Val("retained_bytes", strconv.Itoa(after-before)) + if after-before > largest+allocated/4+1024 { + t.Fail("memory not released, " + strconv.Itoa(after-before) + " of " + strconv.Itoa(allocated) + " bytes retained") + } +} + +// noGrowth runs f eight times and fails if memory use grows on each of the last +// three runs and has grown by more than one and a half structures since the +// second run. A stale pointer can keep a whole structure alive, and which +// pointers are stale differs from run to run, so memory use may jump or wobble. +// A real leak adds a structure on every run, so it keeps going up. +func noGrowth(t *dev.T, what string, size int, f func() bool) { + const runs = 8 + var used [runs]int + for i := range used { + if !f() { + t.Fail(what + " was corrupted") + return + } + used[i] = inUse() + t.Val("in_use_after_run_"+strconv.Itoa(i+1), strconv.Itoa(used[i])) + } + step := size / 4 + last := runs - 1 + rising := used[last]-used[last-1] > step && used[last-1]-used[last-2] > step && used[last-2]-used[last-3] > step + if rising && used[last]-used[1] > size+size/2 { + t.Fail("memory use grew on every run, from " + strconv.Itoa(used[1]) + " to " + strconv.Itoa(used[last]) + " bytes") + } +} + +//go:noinline +func allocAndDrop(n int) { + b := make([]byte, n) + for i := range b { + b[i] = byte(i) + } + sink = b + sink = nil +} + +// numGC returns the number of collections that have run. +// +//go:noinline +func numGC() uint32 { + var m runtime.MemStats + runtime.ReadMemStats(&m) + return uint32(m.NumGC) +} + +// reclaims checks that runtime.GC runs a collection. A leaking collector or none +// at all would make the other tests run out of memory. How much of a dropped +// block stays is only reported, because stale values in registers and the +// stack can keep the last few blocks alive. +func reclaims(t *dev.T) { + free := freeHeap() + t.Val("heap_free_bytes", strconv.Itoa(free)) + t.Val("heap_in_use_bytes", strconv.Itoa(inUse())) + n := free / 8 + if n < 512 { + noReclaim = true + t.Skip("heap too small") + return + } + start := numGC() + runtime.GC() + if numGC() == start { + noReclaim = true + t.Skip("runtime.GC does not run a collection") + return + } + before := inUse() + for i := 0; i < 3; i++ { + allocAndDrop(n + i*16) + } + t.Val("retained_bytes", strconv.Itoa(inUse()-before)) +} + +//go:noinline +func bigBlock(n int) bool { + b := make([]byte, n) + for i := range b { + b[i] = byte(i * 13) + } + for i := range b { + if b[i] != byte(i*13) { + return false + } + } + return true +} + +func largeBlock(t *dev.T) { + n := budget(t, 3) + if n == 0 { + return + } + before := inUse() + if !bigBlock(n) { + t.Fail("contents of a " + strconv.Itoa(n) + " byte block changed") + } + t.Val("retained_bytes", strconv.Itoa(inUse()-before)) +} + +type node struct { + next *node + id uint32 + pad [4]uint32 +} + +//go:noinline +func listTest(n int) bool { + var head *node + for i := 0; i < n; i++ { + head = &node{next: head, id: uint32(i) * 2654435761} + } + for i := n - 1; i >= 0; i-- { + if head == nil || head.id != uint32(i)*2654435761 { + return false + } + head = head.next + } + return head == nil +} + +func linkedList(t *dev.T) { + b := budget(t, 4) + if b == 0 { + return + } + n := b / 40 + noGrowth(t, "linked list", n*32, func() bool { return listTest(n) }) +} + +//go:noinline +func growTest(limit int) bool { + var s []byte + for i := 0; i < limit; i++ { + s = append(s, byte(i*7)) + } + for i := range s { + if s[i] != byte(i*7) { + return false + } + } + return len(s) == limit +} + +func sliceGrow(t *dev.T) { + b := budget(t, 4) + if b == 0 { + return + } + limit := b / 3 + noGrowth(t, "appended data", limit*2, func() bool { return growTest(limit) }) +} + +//go:noinline +func stringsCheck(count int) bool { + var sb strings.Builder + for i := 0; i < count; i++ { + sb.WriteString(strconv.Itoa(i)) + sb.WriteByte(',') + } + s := sb.String() + pos := 0 + for i := 0; i < count; i++ { + d := strconv.Itoa(i) + "," + if pos+len(d) > len(s) || s[pos:pos+len(d)] != d { + return false + } + pos += len(d) + } + return pos == len(s) +} + +// stringsTest builds a long decimal list. +func stringsTest(t *dev.T) { + b := budget(t, 6) + if b == 0 { + return + } + count := b / 24 + before := inUse() + if !stringsCheck(count) { + t.Fail("string builder output was wrong") + } + checkLeak(t, before, b, b/2) +} + +//go:noinline +func garbage(n int) { + sink = make([]byte, n) +} + +func gcCycles(t *dev.T) { + b := budget(t, 8) + if b == 0 { + return + } + size := 64 + 64*(b/4096%8) + if size > b/4 { + size = b / 4 + } + runtime.GC() + start := stats() + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + garbage(size) + m := stats() + if m.TotalAlloc-start.TotalAlloc > 3*m.HeapSys && m.NumGC > start.NumGC { + break + } + } + sink = nil + end := stats() + t.Val("collections", strconv.Itoa(int(end.NumGC-start.NumGC))) + t.Val("allocated_bytes", strconv.Itoa(int(end.TotalAlloc-start.TotalAlloc))) + if end.NumGC == start.NumGC { + allocated := end.TotalAlloc - start.TotalAlloc + if end.HeapSys > 8*allocated { + t.Skip("the heap is too large to reach a collection in a few seconds") + return + } + t.Fail("no collection ran while allocating " + strconv.Itoa(int(allocated)) + " bytes") + } +} diff --git a/tools/qbs/main.go b/tools/qbs/main.go new file mode 100644 index 0000000000..d17d9f6b72 --- /dev/null +++ b/tools/qbs/main.go @@ -0,0 +1,605 @@ +// qbs runs quick, broad, shallow tests on a connected board. It flashes a +// program for each class of tests, collects the results over serial and +// checks them. +// +// go run ./tools/qbs -target pico +// go run ./tools/qbs -target arduino-uno-r4-wifi timing gc +// +// Classes: +// +// timing sleep accuracy and the device clock against the host clock +// compute results of the same computations on the device and on the host +// gc allocation, collection and fragmentation checks +// serial echo, RX overrun, line and byte transfer on the default serial port +// +// With -emulator the programs run in the emulator of the target through +// tinygo run, for example -target wasip1, and no board is needed. +package main + +import ( + "flag" + "fmt" + "math" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strconv" + "strings" + "time" + + "go.bug.st/serial" + + "github.com/tinygo-org/tinygo/tools/qbs/cases" +) + +type tick struct { + device float64 // seconds on the device clock + host time.Time +} + +type testResult struct { + name string + status string + detail string + elapsed time.Duration + vals [][2]string + logs []string + ticks []tick + stray []string + raw map[string][]byte + // hostFail is set by a driver that found a problem while it ran. + hostFail string +} + +// driver runs on the host when a test begins. It may send data to the device +// and read from it. +type driver func(l *link, t *testResult) + +type classResult struct { + class string + tests []*testResult + done bool + err string + raw []byte +} + +type link struct { + port conn + buf []byte + log []byte +} + +func (l *link) fill() { + var tmp [512]byte + n, _ := l.port.Read(tmp[:]) + if n > 0 { + l.buf = append(l.buf, tmp[:n]...) + l.log = append(l.log, tmp[:n]...) + } +} + +// readLine returns the next line and the time it was complete. +func (l *link) readLine(timeout time.Duration) (string, time.Time, bool) { + deadline := time.Now().Add(timeout) + for { + for i, c := range l.buf { + if c == '\n' { + line := strings.TrimRight(string(l.buf[:i]), "\r") + l.buf = l.buf[i+1:] + return cleanConsole(line), time.Now(), true + } + } + if time.Now().After(deadline) { + return "", time.Time{}, false + } + l.fill() + } +} + +// readExact returns n bytes, or what arrived before the timeout. +func (l *link) readExact(n int, timeout time.Duration) []byte { + deadline := time.Now().Add(timeout) + for len(l.buf) < n && time.Now().Before(deadline) { + l.fill() + } + if len(l.buf) < n { + n = len(l.buf) + } + out := append([]byte(nil), l.buf[:n]...) + l.buf = l.buf[n:] + return out +} + +var ansiEscape = regexp.MustCompile("\x1b\\[[0-9;]*m") + +// cleanConsole removes what the simavr console adds to a line. It colours the +// line and shows the end of line characters as two dots. +func cleanConsole(line string) string { + if !strings.Contains(line, "\x1b[") { + return line + } + line = ansiEscape.ReplaceAllString(line, "") + return strings.TrimSuffix(line, "..") +} + +func openPort(name string) (*link, error) { + deadline := time.Now().Add(15 * time.Second) + var err error + for time.Now().Before(deadline) { + var p serial.Port + p, err = serial.Open(name, &serial.Mode{BaudRate: 115200}) + if err == nil { + p.SetReadTimeout(50 * time.Millisecond) + return &link{port: p}, nil + } + time.Sleep(500 * time.Millisecond) + } + return nil, fmt.Errorf("cannot open %s: %v", name, err) +} + +func findPort() string { + for _, pattern := range []string{"/dev/cu.usbmodem*", "/dev/ttyACM*", "/dev/cu.usbserial*", "/dev/ttyUSB*"} { + if ports, _ := filepath.Glob(pattern); len(ports) > 0 { + return ports[0] + } + } + return "" +} + +type options struct { + tinygo string + target string + // port is the serial port the results are read from. + port string + // flashPort is passed to tinygo flash. It is empty unless the user chose + // a port, so that tinygo finds the board itself. + flashPort string + pathPrepend string + // noInput is set for emulated targets that cannot receive input. + noInput bool + // pty runs the emulator under a pseudo terminal, so that emulators that + // buffer their output when it is a pipe print each line at once. + pty bool + // scheduler overrides the scheduler the programs are built with. + scheduler string + // gc overrides the garbage collector the programs are built with. + gc string +} + +// flash returns skip with a reason when the program does not fit on the target. +// It waits up to 90 seconds for the drive of a UF2 bootloader to be mounted. +func flash(o options, dir string) (skip string, err error) { + args := []string{"flash", "-target", o.target, "-timeout", "90s"} + if o.flashPort != "" { + args = append(args, "-port", o.flashPort) + } + cmd := exec.Command(o.tinygo, append(args, dir)...) + cmd.Dir = dir + if o.pathPrepend != "" { + cmd.Env = append(os.Environ(), "PATH="+o.pathPrepend+string(os.PathListSeparator)+os.Getenv("PATH")) + } + out, runErr := cmd.CombinedOutput() + if runErr == nil { + return "", nil + } + text := string(out) + if reason := fitFailure(text); reason != "" { + return reason, nil + } + if len(text) > 1500 { + text = text[len(text)-1500:] + } + return "", fmt.Errorf("flash failed: %v\n%s", runErr, text) +} + +// classTimeout limits how long a class may take in all. A program can keep +// printing, for example garbage after it runs out of memory, so the silence +// timeout alone does not end it. +var classTimeout = 10 * time.Minute + +// readyTimeout is how long to wait for the first banner of a program. An +// emulated program is compiled first, which takes longer. +var readyTimeout = 25 * time.Second + +// collect runs the handshake and reads results until the device reports the end. +func collect(l *link, class string, drivers map[string]driver, noInput bool) *classResult { + r := &classResult{class: class} + defer func() { r.raw = l.log }() + + // A program that cannot receive input starts without the host, so wait + // for its first line and send nothing. + banner := "@@READY qbs " + class + if noInput { + banner = "@@GO " + class + } + ready := false + deadline := time.Now().Add(readyTimeout) + for time.Now().Before(deadline) { + line, _, ok := l.readLine(time.Second) + if ok && strings.HasPrefix(line, banner) { + ready = true + break + } + // An emulated program that already ended will not print a banner. + if e, isPipe := l.port.(interface{ exited() bool }); isPipe && e.exited() { + break + } + } + if !ready { + r.err = "no " + banner + " line, the program printed nothing" + return r + } + if !noInput { + l.port.Write([]byte("G")) + } + + // The first @@GO is the banner when the program does not wait for the host. + started := noInput + var cur *testResult + classDeadline := time.Now().Add(classTimeout) + for { + if time.Now().After(classDeadline) { + where := "before the first test" + if cur != nil { + where = "in test " + cur.name + } + r.err = "the class did not finish within " + classTimeout.String() + ", it was " + where + return r + } + line, at, ok := l.readLine(60 * time.Second) + if !ok { + where := "before the first test" + if cur != nil { + where = "in test " + cur.name + } + r.err = "no output for 60s " + where + return r + } + f := strings.Fields(line) + if len(f) == 0 || !strings.HasPrefix(f[0], "@@") { + if cur != nil { + cur.stray = append(cur.stray, line) + } + continue + } + rest := strings.TrimSpace(strings.TrimPrefix(line, f[0])) + switch f[0] { + case "@@GO": + // A second one means the program restarted, for example after a crash. + if started { + where := "before the first test" + if cur != nil { + where = "in test " + cur.name + } + r.err = "the program restarted " + where + return r + } + started = true + case "@@BEGIN": + cur = &testResult{name: rest, raw: map[string][]byte{}} + r.tests = append(r.tests, cur) + if d := drivers[cur.name]; d != nil { + d(l, cur) + } + case "@@LOG": + if cur != nil { + cur.logs = append(cur.logs, rest) + } + case "@@VAL": + if cur != nil && len(f) >= 2 { + cur.vals = append(cur.vals, [2]string{f[1], strings.TrimSpace(strings.TrimPrefix(rest, f[1]))}) + } + case "@@RAW": + if cur != nil && len(f) == 3 { + n, err := strconv.Atoi(f[2]) + if err == nil { + cur.raw[f[1]] = l.readExact(n, 10*time.Second) + } + } + case "@@TICK": + if cur != nil && len(f) == 3 { + if us, err := strconv.ParseFloat(f[2], 64); err == nil { + cur.ticks = append(cur.ticks, tick{device: us / 1e6, host: at}) + } + } + case "@@RESULT": + if cur != nil && len(f) >= 4 && f[1] == cur.name { + cur.status = f[2] + us, _ := strconv.ParseInt(f[3], 10, 64) + cur.elapsed = time.Duration(us) * time.Microsecond + cur.detail = strings.TrimSpace(strings.Join(f[4:], " ")) + } + case "@@END": + r.done = true + return r + } + } +} + +func (t *testResult) val(key string) (string, bool) { + for _, kv := range t.vals { + if kv[0] == key { + return kv[1], true + } + } + return "", false +} + +func (t *testResult) fail(format string, args ...interface{}) { + t.status = "FAIL" + t.detail = fmt.Sprintf(format, args...) +} + +// emulated is set with -emulator. An emulator does not keep wall clock time. +var emulated bool + +// evalTiming compares the device clock with the host clock. +func evalTiming(r *classResult) { + for _, t := range r.tests { + if t.name != "clock-rate" || t.status != "PASS" { + continue + } + if len(t.ticks) < 20 { + t.fail("only %d of 30 ticks arrived", len(t.ticks)) + continue + } + var mx, my float64 + t0 := t.ticks[0].host + for _, k := range t.ticks { + mx += k.host.Sub(t0).Seconds() + my += k.device + } + n := float64(len(t.ticks)) + mx /= n + my /= n + var sxy, sxx float64 + for _, k := range t.ticks { + x := k.host.Sub(t0).Seconds() - mx + sxy += x * (k.device - my) + sxx += x * x + } + slope := sxy / sxx + if emulated && slope > 10 { + t.status = "SKIP" + t.detail = fmt.Sprintf("the device clock runs %.0f times faster than the host clock, it is probably simulated", slope) + continue + } + t.detail = fmt.Sprintf("device clock runs at %.4f of the host clock", slope) + if math.Abs(slope-1) > 0.02 { + if emulated { + t.status = "SKIP" + t.detail = fmt.Sprintf("the emulated clock runs at %.4f of the host clock", slope) + continue + } + t.fail("device clock runs at %.4f of the host clock", slope) + } + } +} + +// compareCase compares a device result with the host result. +func compareCase(c cases.Case, got, want string) (bool, string) { + if got == want { + return true, "" + } + if c.ULP == 0 { + return false, "device " + shorten(got) + " host " + shorten(want) + } + g, w := strings.Fields(got), strings.Fields(want) + if len(g) != len(w) { + return false, fmt.Sprintf("device returned %d values, host %d", len(g), len(w)) + } + var worst uint64 + for i := range g { + a, errA := strconv.ParseUint(g[i], 16, 64) + b, errB := strconv.ParseUint(w[i], 16, 64) + if errA != nil || errB != nil { + return false, "device " + shorten(got) + " host " + shorten(want) + } + d := a - b + if b > a { + d = b - a + } + if d > worst { + worst = d + } + if d > c.ULP { + return false, fmt.Sprintf("value %d differs by %d ULP (limit %d)", i, d, c.ULP) + } + } + return true, fmt.Sprintf("within %d ULP", worst) +} + +func shorten(s string) string { + if len(s) > 60 { + return s[:57] + "..." + } + return s +} + +// evalCompute runs the same cases on the host and compares. +func evalCompute(r *classResult) { + byName := map[string]cases.Case{} + for _, c := range cases.All() { + byName[c.Name] = c + } + for _, t := range r.tests { + c, ok := byName[t.name] + if !ok || t.status != "PASS" { + continue + } + got, ok := t.val("result") + if !ok { + t.fail("device sent no result") + continue + } + match, why := compareCase(c, got, c.Fn()) + if !match { + t.fail("%s", why) + } else { + t.detail = why + } + } +} + +var classes = []struct { + name string + drivers map[string]driver + eval func(*classResult) + needsInput bool +}{ + {"timing", nil, evalTiming, false}, + {"compute", nil, evalCompute, false}, + {"gc", nil, nil, false}, + {"serial", serialDrivers, evalSerial, true}, +} + +func main() { + target := flag.String("target", "", "tinygo target (required)") + tinygo := flag.String("tinygo", "tinygo", "tinygo binary to use") + port := flag.String("port", "", "serial port (default: first USB serial port found)") + pathPrepend := flag.String("path-prepend", "", "directory to put first on PATH when flashing") + verbose := flag.Bool("v", false, "print the values and messages each test sent") + dump := flag.Bool("dump", false, "print raw serial data when a class fails to finish") + emulate := flag.Bool("emulator", false, "run the programs in the emulator of the target instead of flashing a board") + usePTY := flag.Bool("pty", false, "with -emulator, run the emulator under a pseudo terminal, which simavr needs to print lines as they happen") + scheduler := flag.String("scheduler", "", "with -emulator, build the programs with this scheduler, for example none") + flag.DurationVar(&classTimeout, "class-timeout", classTimeout, "longest time one class may take") + gcFlag := flag.String("gc", "", "with -emulator, build the programs with this garbage collector, for example precise") + noInput := flag.Bool("noinput", false, "with -emulator, the target cannot receive input, so run only the classes that need none") + flag.Parse() + + if *target == "" { + fmt.Fprintln(os.Stderr, "missing -target") + os.Exit(2) + } + flashPort := *port + emulated = *emulate + if *port == "" && !*emulate { + *port = findPort() + } + if *port == "" && !*emulate { + fmt.Fprintln(os.Stderr, "no serial port found, use -port") + os.Exit(2) + } + want := map[string]bool{} + for _, name := range flag.Args() { + want[name] = true + } + for name := range want { + known := false + for _, c := range classes { + known = known || c.name == name + } + if !known { + fmt.Fprintf(os.Stderr, "unknown class %q\n", name) + os.Exit(2) + } + } + + _, file, _, _ := runtime.Caller(0) + here := filepath.Dir(file) + o := options{tinygo: *tinygo, target: *target, port: *port, flashPort: flashPort, pathPrepend: *pathPrepend, noInput: *noInput, pty: *usePTY, scheduler: *scheduler, gc: *gcFlag} + + fmt.Printf("port %s, target %s\n", *port, *target) + counts := map[string]int{} + report := func(status, class, name, detail string) { + counts[status]++ + fmt.Printf("%-5s %-8s %-16s %s\n", status, class, name, detail) + } + + for _, c := range classes { + if len(want) > 0 && !want[c.name] { + continue + } + fmt.Printf("\n== %s\n", c.name) + if *noInput && c.needsInput { + report("SKIP", c.name, "(class)", "the target cannot receive input") + continue + } + var l *link + if *emulate { + readyTimeout = 3 * time.Minute + var err error + l, err = startEmulator(o, filepath.Join(here, c.name)) + if err != nil { + if reason := emulatorFailure([]byte(err.Error())); reason != "" { + report("SKIP", c.name, "(class)", reason) + } else { + report("FAIL", c.name, "(emulator)", err.Error()) + } + continue + } + } else { + flashStart := time.Now() + skip, err := flash(o, filepath.Join(here, c.name)) + if took := time.Since(flashStart); took > 15*time.Second { + fmt.Printf("note: flashing took %s\n", took.Round(time.Second)) + } + if err != nil { + report("FAIL", c.name, "(flash)", err.Error()) + continue + } + if skip != "" { + report("SKIP", c.name, "(class)", skip) + continue + } + time.Sleep(time.Second) + l, err = openPort(*port) + if err != nil { + report("FAIL", c.name, "(port)", err.Error()) + continue + } + } + r := collect(l, c.name, c.drivers, *noInput) + l.port.Close() + if *emulate { + if reason := emulatorFailure(r.raw); reason != "" { + report("SKIP", c.name, "(class)", reason) + continue + } + } + + if c.eval != nil { + c.eval(r) + } + for _, t := range r.tests { + status := t.status + detail := t.detail + if status == "" { + status = "FAIL" + detail = "no result, the device stopped in this test" + if len(t.stray) > 0 { + detail += ": " + shorten(strings.Join(t.stray, " | ")) + } + } else if status == "PASS" && detail == "" { + detail = t.elapsed.Round(time.Millisecond).String() + } + report(status, c.name, t.name, detail) + if *verbose { + for _, kv := range t.vals { + fmt.Printf(" %s = %s\n", kv[0], shorten(kv[1])) + } + for _, m := range t.logs { + fmt.Printf(" log: %s\n", m) + } + } + } + if r.err != "" { + report("FAIL", c.name, "(class)", r.err) + if *dump { + raw := r.raw + if len(raw) > 400 { + raw = raw[len(raw)-400:] + } + fmt.Printf("raw serial data: %q\n", raw) + } + } + } + + fmt.Printf("\n%d pass, %d fail, %d skip\n", counts["PASS"], counts["FAIL"], counts["SKIP"]) + if counts["FAIL"] > 0 { + os.Exit(1) + } +} diff --git a/tools/qbs/serial.go b/tools/qbs/serial.go new file mode 100644 index 0000000000..246693b63d --- /dev/null +++ b/tools/qbs/serial.go @@ -0,0 +1,238 @@ +package main + +import ( + "bytes" + "encoding/hex" + "fmt" + "math/rand" + "regexp" + "strconv" + "strings" + "time" +) + +var serialDrivers = map[string]driver{ + "echo": driveEcho, + "burst": driveBurst, + "conc": driveConc, +} + +// driveEcho sends 666 bytes in three phases and checks that they come back. +func driveEcho(l *link, t *testResult) { + rng := rand.New(rand.NewSource(1)) + random := func(n int) []byte { + b := make([]byte, n) + for i := range b { + b[i] = byte(rng.Intn(256)) + } + return b + } + phase := func(label string, chunks [][]byte, gap time.Duration) string { + var sent []byte + for _, c := range chunks { + l.port.Write(c) + sent = append(sent, c...) + time.Sleep(gap) + } + got := l.readExact(len(sent), 5*time.Second) + if !bytes.Equal(got, sent) { + n := 0 + for n < len(got) && n < len(sent) && got[n] == sent[n] { + n++ + } + return fmt.Sprintf("%s: sent %d got %d bytes, first difference at %d", label, len(sent), len(got), n) + } + return "" + } + + var lines [][]byte + for i := 0; i < 10; i++ { + lines = append(lines, []byte(fmt.Sprintf("hello %d\r\n", i))) + } + var paced [][]byte + for i := 0; i < 16; i++ { + paced = append(paced, random(32)) + } + for _, p := range []struct { + label string + chunks [][]byte + gap time.Duration + }{ + {"small", lines, 20 * time.Millisecond}, + {"64B block", [][]byte{random(64)}, 50 * time.Millisecond}, + {"512B paced", paced, 10 * time.Millisecond}, + } { + if msg := phase(p.label, p.chunks, p.gap); msg != "" { + t.hostFail = msg + return + } + } +} + +const burstRecovery = "RECOVERY-OK-0123" + +// driveBurst sends more than the RX buffer holds while the device is not +// reading, then sends a short message after the device has read the buffer. +func driveBurst(l *link, t *testResult) { + pat := make([]byte, 4096) + for i := range pat { + pat[i] = byte(i % 251) + } + l.port.Write(pat) + time.Sleep(2500 * time.Millisecond) + l.port.Write([]byte(burstRecovery)) +} + +// driveConc sends 512 bytes while the device prints from goroutines. +func driveConc(l *link, t *testResult) { + payload := make([]byte, 512) + for i := range payload { + payload[i] = byte(i) + } + for i := 0; i < len(payload); i += 16 { + l.port.Write(payload[i : i+16]) + time.Sleep(10 * time.Millisecond) + } +} + +func evalSerial(r *classResult) { + for _, t := range r.tests { + if t.status != "PASS" { + continue + } + var msg, ok string + switch t.name { + case "echo": + msg, ok = checkEcho(t) + case "burst": + msg, ok = checkBurst(t) + case "seq": + msg, ok = checkSeq(t) + case "bytes": + msg, ok = checkBytes(t) + case "conc": + msg, ok = checkConc(t) + default: + continue + } + if msg != "" { + t.fail("%s", msg) + } else { + t.detail = ok + } + } +} + +func checkEcho(t *testResult) (string, string) { + if t.hostFail != "" { + return t.hostFail, "" + } + if v, _ := t.val("echoed"); v != "666" { + return "device echoed " + v + " of 666 bytes", "" + } + return "", "echoed 10 lines, 64 bytes and 512 paced random bytes" +} + +func checkBurst(t *testResult) (string, string) { + nstr, _ := t.val("buf") + n, err := strconv.Atoi(nstr) + if err != nil { + return "no buffer size from the device", "" + } + data, _ := t.val("data") + got, err := hex.DecodeString(data) + if err != nil || len(got) != n { + return "device data does not match its buffer size", "" + } + if n == 0 { + return "no bytes received at all", "" + } + note := fmt.Sprintf("RX buffer held %d of 4096 bytes", n) + for i, b := range got { + if b != byte(i%251) { + return fmt.Sprintf("%s, but the data is not a prefix of the burst (first bad index %d)", note, i), "" + } + } + rec, _ := t.val("rec") + recBytes, _ := hex.DecodeString(rec) + if string(recBytes) != burstRecovery { + return fmt.Sprintf("receiving stopped after the overrun, got %q", recBytes), "" + } + return "", note + ", receive works after the overrun" +} + +func checkSeq(t *testResult) (string, string) { + var lines []string + for _, s := range t.stray { + if strings.HasPrefix(s, "S ") { + lines = append(lines, s) + } + } + if len(lines) != 2000 { + return fmt.Sprintf("got %d of 2000 lines", len(lines)), "" + } + for i, s := range lines { + want := fmt.Sprintf("S %d %d", i, i*7919%10007) + if s != want { + return fmt.Sprintf("line %d: got %q want %q", i, s, want), "" + } + } + return "", "2000 lines in order" +} + +func checkBytes(t *testResult) (string, string) { + for k := 0; k < 4; k++ { + want := make([]byte, 256) + for i := range want { + switch k { + case 0: + want[i] = byte(i) + case 1: + want[i] = byte(255 - i) + case 2: + want[i] = 0 + default: + if i%2 == 0 { + want[i] = 0xAA + } else { + want[i] = 0x55 + } + } + } + got := t.raw["p"+strconv.Itoa(k)] + if !bytes.Equal(got, want) { + return fmt.Sprintf("pattern %d differs (got %d bytes)", k, len(got)), "" + } + } + return "", "4 patterns of 256 bytes, all byte values intact" +} + +var concLine = regexp.MustCompile(`^G (\d) (\d+)$`) + +func checkConc(t *testResult) (string, string) { + last := [3]int{-1, -1, -1} + count := 0 + for _, s := range t.stray { + if s == "" { + continue + } + m := concLine.FindStringSubmatch(s) + if m == nil { + return fmt.Sprintf("torn or garbled line %q", s), "" + } + gid, _ := strconv.Atoi(m[1]) + k, _ := strconv.Atoi(m[2]) + if gid > 2 || k != last[gid]+1 { + return fmt.Sprintf("goroutine %d out of order: %d after %d", gid, k, last[gid]), "" + } + last[gid] = k + count++ + } + if count != 600 || last != [3]int{199, 199, 199} { + return fmt.Sprintf("saw %d goroutine lines, last %v", count, last), "" + } + if v, _ := t.val("rx"); v != "512" { + return "device counted " + v + " of 512 RX bytes", "" + } + return "", "600 interleaved lines intact, 512/512 RX bytes counted" +} diff --git a/tools/qbs/serial/conc.go b/tools/qbs/serial/conc.go new file mode 100644 index 0000000000..a0068755ef --- /dev/null +++ b/tools/qbs/serial/conc.go @@ -0,0 +1,62 @@ +//go:build tinygo && !scheduler.none + +package main + +import ( + "strconv" + "sync" + "time" + + "github.com/tinygo-org/tinygo/tools/qbs/dev" +) + +var sink []byte + +func init() { + extraTests = append(extraTests, dev.Test{Name: "conc", Run: conc}) +} + +func worker(id int, wg *sync.WaitGroup) { + defer wg.Done() + for k := 0; k < 200; k++ { + sink = make([]byte, 256) + println("G", id, k) + time.Sleep(time.Millisecond) + } +} + +// conc prints from goroutines while the host sends bytes. +func conc(t *dev.T) { + var wg sync.WaitGroup + for id := 0; id < 3; id++ { + wg.Add(1) + go worker(id, &wg) + } + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + + rx := 0 + finished := false + for !finished { + for dev.Serial.Buffered() > 0 { + dev.Serial.ReadByte() + rx++ + } + select { + case <-done: + finished = true + default: + time.Sleep(time.Millisecond) + } + } + // Let the host finish sending, then count what is left. + time.Sleep(500 * time.Millisecond) + for dev.Serial.Buffered() > 0 { + dev.Serial.ReadByte() + rx++ + } + t.Val("rx", strconv.Itoa(rx)) +} diff --git a/tools/qbs/serial/main.go b/tools/qbs/serial/main.go new file mode 100644 index 0000000000..4ce62fe84f --- /dev/null +++ b/tools/qbs/serial/main.go @@ -0,0 +1,93 @@ +//go:build tinygo + +package main + +import ( + "strconv" + "time" + + "github.com/tinygo-org/tinygo/tools/qbs/dev" +) + +// extraTests holds tests that not every target can build. +var extraTests []dev.Test + +func main() { + tests := []dev.Test{ + {Name: "echo", Run: echo}, + {Name: "burst", Run: burst}, + {Name: "seq", Run: seq}, + {Name: "bytes", Run: bytes}, + } + dev.Main("serial", append(tests, extraTests...)) +} + +// echoLen is the number of bytes the host sends to the echo test. +const echoLen = 666 + +// echo sends back every byte it receives. +func echo(t *dev.T) { + n := 0 + last := time.Now() + for n < echoLen && time.Since(last) < 3*time.Second { + if dev.Serial.Buffered() == 0 { + time.Sleep(time.Millisecond) + continue + } + b, _ := dev.Serial.ReadByte() + dev.Serial.WriteByte(b) + n++ + last = time.Now() + } + t.Val("echoed", strconv.Itoa(n)) +} + +// burst does not read while the host sends more than the RX buffer holds. +func burst(t *dev.T) { + time.Sleep(1500 * time.Millisecond) + var got []byte + for dev.Serial.Buffered() > 0 { + b, _ := dev.Serial.ReadByte() + got = append(got, b) + } + t.Val("buf", strconv.Itoa(len(got))) + t.Val("data", dev.Hex(got)) + + // The port must keep receiving after the overrun. + t.Val("rec", dev.Hex(dev.ReadN(16, 5*time.Second))) +} + +// seq prints numbered lines as fast as it can. +func seq(t *dev.T) { + for i := 0; i < 2000; i++ { + println("S", i, i*7919%10007) + } +} + +func bytePattern(k int) []byte { + b := make([]byte, 256) + for i := range b { + switch k { + case 0: + b[i] = byte(i) + case 1: + b[i] = byte(255 - i) + case 2: + b[i] = 0x00 + default: + if i%2 == 0 { + b[i] = 0xAA + } else { + b[i] = 0x55 + } + } + } + return b +} + +// bytes sends every byte value in four patterns. +func bytes(t *dev.T) { + for k := 0; k < 4; k++ { + t.Raw("p"+strconv.Itoa(k), bytePattern(k)) + } +} diff --git a/tools/qbs/timing/main.go b/tools/qbs/timing/main.go new file mode 100644 index 0000000000..ed5c862a4d --- /dev/null +++ b/tools/qbs/timing/main.go @@ -0,0 +1,104 @@ +//go:build tinygo + +package main + +import ( + "runtime/volatile" + "strconv" + "time" + + "github.com/tinygo-org/tinygo/tools/qbs/dev" +) + +func main() { + dev.Main("timing", []dev.Test{ + {Name: "sleep-100ms", Run: sleep100}, + {Name: "sleep-1ms", Run: sleep1}, + {Name: "monotonic", Run: monotonic}, + {Name: "busy-loop", Run: busyLoop}, + {Name: "clock-rate", Run: clockRate}, + }) +} + +func sleep100(t *dev.T) { + var min, max time.Duration + for i := 0; i < 20; i++ { + start := time.Now() + time.Sleep(100 * time.Millisecond) + d := time.Since(start) + if i == 0 || d < min { + min = d + } + if d > max { + max = d + } + } + t.Val("min_us", strconv.Itoa(int(min.Microseconds()))) + t.Val("max_us", strconv.Itoa(int(max.Microseconds()))) + if min < 95*time.Millisecond { + t.Fail("a 100ms sleep returned after " + min.String()) + } + if max > 200*time.Millisecond { + t.Fail("a 100ms sleep took " + max.String()) + } +} + +func sleep1(t *dev.T) { + start := time.Now() + for i := 0; i < 100; i++ { + time.Sleep(time.Millisecond) + } + d := time.Since(start) + t.Val("total_us", strconv.Itoa(int(d.Microseconds()))) + if d < 90*time.Millisecond { + t.Fail("100 sleeps of 1ms took only " + d.String()) + } + if d > time.Second { + t.Fail("100 sleeps of 1ms took " + d.String()) + } +} + +func monotonic(t *dev.T) { + start := time.Now() + prev := start + calls := 0 + for time.Since(start) < 50*time.Millisecond && calls < 5000000 { + now := time.Now() + if now.Sub(prev) < 0 { + t.Fail("time.Now went backwards") + return + } + prev = now + calls++ + } + if time.Since(start) == 0 { + t.Skip("the clock does not advance while the program runs") + return + } + t.Val("calls_per_ms", strconv.Itoa(calls/50)) +} + +func busyLoop(t *dev.T) { + start := time.Now() + // The volatile variable keeps the compiler from removing the loop or + // moving it past the clock reads. + var v volatile.Register32 + v.Set(2463534242) + for i := 0; i < 100000; i++ { + x := v.Get() + x ^= x << 13 + x ^= x >> 17 + x ^= x << 5 + v.Set(x) + } + t.Val("us_per_100k_iterations", strconv.Itoa(int(time.Since(start).Microseconds()))) +} + +// clockRate prints the device clock so the host can compare it with its own. +func clockRate(t *dev.T) { + start := time.Now() + for i := 0; i < 30; i++ { + time.Sleep(100 * time.Millisecond) + println("@@TICK", i, time.Since(start).Microseconds()) + } +}