tools: add qbs, quick broad shallow tests for connected boards

tools/qbs/README.md

@@ -0,0 +1,74 @@
+# qbs
+
+Quick, broad, shallow tests for a connected board. Each test is small and
+runs on every target. The aim is to find obvious problems on many boards
+quickly, not to test any one feature in depth.
+
+    go run ./tools/qbs -target <target>
+
+The tool flashes one program per class with `tinygo flash`, collects results
+over the board's serial port and checks them. The tool builds with the
+standard Go compiler. The programs it flashes build with TinyGo.
+
+Classes:
+
+- timing: sleep accuracy, a monotonic clock, loop speed, and the device clock
+  against the host clock
+- compute: the same computations (integers, bits, sorting, maps, hashes,
+  encodings, strings, float32 and float64, math, strconv, fmt) run on the
+  device and on the host and the results are compared
+- gc: large blocks, linked lists, slice and string growth, map churn,
+  fragmentation and collection cycles, sized from the free heap. The map and
+  fragmentation tests are not built for AVR. A conservative collector can keep
+  a whole linked structure alive through one stale pointer, so these tests
+  fail only when memory use keeps growing over repeated runs.
+- serial: the default serial port. Bytes sent to the board come back
+  unchanged (echo), overflowing the RX buffer keeps the first bytes and
+  receiving still works (burst), 2000 numbered lines arrive in order (seq), all
+  256 byte values arrive intact (bytes), and output and input work together
+  with goroutines and allocation (conc, not built on targets without a
+  scheduler)
+
+Use `-v` to see the values each test sent, `-port` if the board is not found,
+`-tinygo` for a different tinygo binary and `-path-prepend` to put a flashing
+tool first on PATH. Names after the flags select classes.
+
+Use `-emulator` to run the programs in the emulator of the target instead of
+flashing a board, for example `go run ./tools/qbs -target wasip1 -emulator`.
+The runner then starts `tinygo run` for each class and talks to the program
+through its standard input and output. The emulator, such as wasmtime or QEMU,
+must be installed.
+
+Some emulated targets cannot receive input, for example riscv-qemu and
+cortex-m-qemu. Add `-noinput` for them. The programs then start without the
+host and the serial class is skipped.
+
+simavr holds back its output when it is not a terminal and wraps each line in
+colour codes, so add `-pty`. The tasks scheduler of the simavr target breaks an
+explicit `runtime.GC()`, so build without a scheduler:
+
+    go run ./tools/qbs -target simavr -emulator -noinput -pty -scheduler none
+
+`-gc` builds the programs with another garbage collector, for example
+`-gc precise`. Use it for simavr. The default conservative collector takes
+many values on a 16 bit AVR for addresses of the heap, and the gc tests then run
+out of memory. `-class-timeout` limits how long one class may take, 10 minutes
+by default. A program that prints garbage after it runs out of memory would
+otherwise never end.
+
+A class is skipped when its program does not fit on the target. A test is
+skipped when the target cannot run it, for example when the garbage collector
+does not reclaim memory.
+
+## Adding a test
+
+- A computation for the compute class goes in `cases`. It must give the same
+  result on the host and the device. Use `ULP` for float results that may
+  differ in the last bits.
+- Any other test is a `dev.Test` in the program of its class. Use `t.Fail`,
+  `t.Skip`, `t.Val` and `t.Raw`. A test that needs the host to judge the
+  result sends values, ticks or raw bytes and gets a check in the host code.
+  A test that needs the host to send data gets a driver, which runs on the
+  host when the test begins (see `serial.go`).
+- A new class is a new directory with a program that calls `dev.Main`, and an
+  entry in the class list in `main.go`.

tools/qbs/cases/cases.go

@@ -0,0 +1,68 @@
+// Package cases has deterministic computations that run on the device and on
+// the host. The host compares the two results.
+package cases
+
+import "strconv"
+
+// Case is one computation. Fn returns a short string that describes the result.
+type Case struct {
+	Name string
+	Fn   func() string
+	// ULP is the allowed difference in units in the last place when Fn
+	// returns space separated hex float bits. Zero means an exact match.
+	ULP uint64
+	// Heap is the number of bytes of heap the case needs. The device skips
+	// the case when less is free.
+	Heap int
+}
+
+var all []Case
+
+func register(name string, ulp uint64, fn func() string) {
+	all = append(all, Case{Name: name, Fn: fn, ULP: ulp})
+}
+
+// needsHeap records how much heap the case registered last needs.
+func needsHeap(bytes int) {
+	all[len(all)-1].Heap = bytes
+}
+
+// All returns the cases that are built for this target.
+func All() []Case {
+	return all
+}
+
+// rng is a xorshift32 generator.
+type rng uint32
+
+func (r *rng) next() uint32 {
+	x := uint32(*r)
+	x ^= x << 13
+	x ^= x >> 17
+	x ^= x << 5
+	*r = rng(x)
+	return x
+}
+
+func randBytes(n int, seed uint32) []byte {
+	r := rng(seed)
+	b := make([]byte, n)
+	for i := range b {
+		b[i] = byte(r.next() >> 11)
+	}
+	return b
+}
+
+func hex64(v uint64) string {
+	return strconv.FormatUint(v, 16)
+}
+
+// fnv64 hashes s so that long results stay short.
+func fnv64(s string) uint64 {
+	h := uint64(14695981039346656037)
+	for i := 0; i < len(s); i++ {
+		h ^= uint64(s[i])
+		h *= 1099511628211
+	}
+	return h
+}

tools/qbs/cases/float.go

@@ -0,0 +1,86 @@
+package cases
+
+import (
+	"math"
+	"strconv"
+	"strings"
+)
+
+// The explicit conversions stop the host compiler from fusing a multiply and
+// an add, which the soft float code on the device does not do.
+
+func init() {
+	register("float64-basic", 0, func() string {
+		r := rng(20)
+		var s float64
+		acc := uint64(0)
+		for i := 0; i < 500; i++ {
+			x := float64(int32(r.next())) / 65536.0
+			y := float64(int32(r.next()))/1024.0 + 0.5
+			s += float64(x * y)
+			s -= float64(x / (y + 1e6))
+			s = float64(s*0.999) + math.Sqrt(math.Abs(x))
+			acc = acc*31 + math.Float64bits(s)
+		}
+		return hex64(acc) + " " + hex64(math.Float64bits(s))
+	})
+
+	register("float32-basic", 0, func() string {
+		r := rng(21)
+		var s float32
+		acc := uint64(0)
+		for i := 0; i < 500; i++ {
+			x := float32(int32(r.next())) / 65536.0
+			y := float32(int32(r.next()))/1024.0 + 0.5
+			s += float32(x * y)
+			s -= float32(x / (y + 1e6))
+			s = float32(s*0.999) + float32(math.Sqrt(math.Abs(float64(x))))
+			acc = acc*31 + uint64(math.Float32bits(s))
+		}
+		return hex64(acc) + " " + hex64(uint64(math.Float32bits(s)))
+	})
+
+	register("math-funcs", 8, func() string {
+		var out []string
+		xs := []float64{0.1, 0.5, 1, 1.5, 2.5, 3.14159, 7, 10.25, 100.5, 0.001}
+		for _, x := range xs {
+			for _, v := range []float64{
+				math.Sin(x), math.Cos(x), math.Tan(x), math.Exp(x / 4), math.Log(x),
+				math.Pow(x, 1.7), math.Atan2(x, 3), math.Sinh(x / 5), math.Cbrt(x),
+			} {
+				out = append(out, hex64(math.Float64bits(v)))
+			}
+		}
+		return strings.Join(out, " ")
+	})
+
+	register("strconv", 0, func() string {
+		r := rng(22)
+		var sb strings.Builder
+		for i := 0; i < 60; i++ {
+			f := math.Float64frombits(uint64(r.next())<<32|uint64(r.next())) / 1e30
+			if math.IsNaN(f) || math.IsInf(f, 0) {
+				f = float64(i)
+			}
+			sb.WriteString(strconv.FormatFloat(f, 'g', -1, 64))
+			sb.WriteString(strconv.FormatFloat(f, 'e', 6, 64))
+			sb.WriteString(strconv.FormatFloat(f, 'f', 3, 32))
+			back, err := strconv.ParseFloat(strconv.FormatFloat(f, 'g', -1, 64), 64)
+			if err != nil || back != f {
+				sb.WriteString("roundtrip-failed")
+			}
+		}
+		for base := 2; base <= 36; base++ {
+			v := int64(int32(r.next()))
+			s := strconv.FormatInt(v, base)
+			back, err := strconv.ParseInt(s, base, 64)
+			if err != nil || back != v {
+				sb.WriteString("int-roundtrip-failed")
+			}
+			sb.WriteString(s)
+		}
+		s := sb.String()
+		return strconv.Itoa(len(s)) + " " + hex64(fnv64(s))
+	})
+	needsHeap(8 << 10)
+}

tools/qbs/cases/fmt.go

@@ -0,0 +1,18 @@
+//go:build !avr
+
+package cases
+
+import "fmt"
+
+func init() {
+	register("fmt", 0, func() string {
+		r := rng(30)
+		s := ""
+		for i := 0; i < 20; i++ {
+			v := int32(r.next())
+			s += fmt.Sprintf("%d %x %08.3f %q %v %5.1f|", v, uint32(v), float64(v)/1000, fmt.Sprint(i), []int{i, i + 1}, float32(i)/3)
+		}
+		return fmt.Sprintf("%d %x", len(s), fnv64(s))
+	})
+	needsHeap(8 << 10)
+}

tools/qbs/cases/hash.go

@@ -0,0 +1,80 @@
+package cases
+
+import (
+	"crypto/md5"
+	"crypto/sha1"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/binary"
+	"encoding/hex"
+	"hash/adler32"
+	"hash/crc32"
+	"hash/fnv"
+	"strings"
+	"unicode/utf8"
+)
+
+func init() {
+	register("crc32", 0, func() string {
+		data := randBytes(1500, 10)
+		castagnoli := crc32.MakeTable(crc32.Castagnoli)
+		sum := crc32.Update(0, castagnoli, data[:700])
+		sum = crc32.Update(sum, castagnoli, data[700:])
+		return hex64(uint64(crc32.ChecksumIEEE(data))) + " " + hex64(uint64(sum))
+	})
+	// Two slicing tables of 8KB each.
+	needsHeap(20 << 10)
+
+	register("adler32-fnv", 0, func() string {
+		data := randBytes(1000, 11)
+		h := fnv.New64a()
+		h.Write(data)
+		h32 := fnv.New32()
+		h32.Write(data)
+		return hex64(uint64(adler32.Checksum(data))) + " " + hex64(h.Sum64()) + " " + hex64(uint64(h32.Sum32()))
+	})
+
+	register("sha256", 0, func() string {
+		sum := sha256.Sum256(randBytes(1000, 12))
+		return hex.EncodeToString(sum[:])
+	})
+	needsHeap(3 << 10)
+
+	register("sha1-md5", 0, func() string {
+		data := randBytes(777, 13)
+		s1 := sha1.Sum(data)
+		m5 := md5.Sum(data)
+		return hex.EncodeToString(s1[:]) + " " + hex.EncodeToString(m5[:])
+	})
+
+	register("encoding", 0, func() string {
+		data := randBytes(300, 14)
+		b64 := base64.StdEncoding.EncodeToString(data)
+		back, err := base64.StdEncoding.DecodeString(b64)
+		if err != nil || string(back) != string(data) {
+			return "base64 round trip failed"
+		}
+		raw := base64.RawURLEncoding.EncodeToString(data[:101])
+		buf := make([]byte, 16)
+		binary.LittleEndian.PutUint64(buf, binary.BigEndian.Uint64(data))
+		binary.BigEndian.PutUint32(buf[8:], binary.LittleEndian.Uint32(data[8:]))
+		return hex64(fnv64(b64)) + " " + hex64(fnv64(raw)) + " " + hex.EncodeToString(buf)
+	})
+	needsHeap(4 << 10)
+
+	register("strings", 0, func() string {
+		var sb strings.Builder
+		r := rng(15)
+		for i := 0; i < 40; i++ {
+			sb.WriteRune(rune(0x20 + r.next()%0x2000))
+			sb.WriteString(" ab,cd;")
+		}
+		s := sb.String()
+		parts := strings.FieldsFunc(s, func(r rune) bool { return r == ',' || r == ';' })
+		joined := strings.Join(parts, "|")
+		up := strings.ToUpper(strings.Repeat(joined[:20], 3))
+		rep := strings.ReplaceAll(s, "ab", "xyz")
+		return hex64(fnv64(joined)) + " " + hex64(fnv64(up)) + " " + hex64(fnv64(rep)) +
+			" " + hex64(uint64(utf8.RuneCountInString(s))) + " " + hex64(uint64(strings.Index(s, "cd;")))
+	})
+}