Clear the false FAILs behind the v1.0.0 gate

dependent.go

@@ -2,10 +2,12 @@ package main
 
 import (
 	"fmt"
-	"github.com/dcadolph/kibble/internal/docblock"
-	kplan "github.com/dcadolph/kibble/internal/plan"
 	"regexp"
 	"strings"
+
+	"github.com/dcadolph/kibble/internal/docblock"
+	kplan "github.com/dcadolph/kibble/internal/plan"
+	"github.com/dcadolph/kibble/internal/shell"
 )
 
 // Failures that may follow from lines which never ran. None of these
@@ -80,11 +82,15 @@ func resolveDependentFailures(run *exampleRun, plan *kplan.Plan) {
 			}
 		}
 	}
+	// Earlier line results per document, for the shared-operand rule, which
+	// reads across the document's steps rather than within one.
+	prior := map[string][]lineResult{}
 	for si := range run.Steps {
 		s := &run.Steps[si]
 		for li := range s.Lines {
 			l := &s.Lines[li]
 			if l.Status != StatusFail {
+				prior[s.Doc] = append(prior[s.Doc], *l)
 				continue
 			}
 			if cited := citedSkipped(l.output, skippedCmds); cited != "" {
@@ -103,9 +109,73 @@ func resolveDependentFailures(run *exampleRun, plan *kplan.Plan) {
 				l.Status = StatusBlocked
 				l.Reason = ReasonDependsOnSkipped
 				l.Detail = fmt.Sprintf("says to run `%s` first, which did not run", need)
+			} else if op, earlier := sharedOperandNotPassed(prior[s.Doc], l.Cmd); op != "" {
+				l.Status = StatusBlocked
+				l.Reason = ReasonDependsOnSkipped
+				l.Detail = fmt.Sprintf("names %s, which `%s` also named and did not pass", op, earlier)
+			}
+			prior[s.Doc] = append(prior[s.Doc], *l)
+		}
+	}
+}
+
+// sharedOperandNotPassed returns a file-shaped operand of a failing line and
+// the first earlier line of the same document that also named it and was
+// skipped, blocked, or stopped by a gap. A query on a database whose load was
+// blocked is the same hole surfacing again. An earlier failure does not count,
+// since a break repeated on several lines of one file is still several
+// reports. Like the other rules here it is not causal: the earlier line still
+// reports, and the later ones stop counting as separate breaks.
+func sharedOperandNotPassed(prior []lineResult, cmd string) (operand, earlier string) {
+	for _, op := range fileOperands(cmd) {
+		for _, p := range prior {
+			if p.Status == StatusVerified || p.Status == StatusFail {
+				continue
+			}
+			for _, pop := range fileOperands(p.Cmd) {
+				if pop == op {
+					return op, docblock.Flatten(p.Cmd)
+				}
+			}
+		}
+	}
+	return "", ""
+}
+
+// reScriptFile matches a source file an interpreter runs, which is the program
+// of a line such as python3 scripts/bench.py rather than data it reads.
+var reScriptFile = regexp.MustCompile(`\.(py|js|mjs|cjs|ts|sh|bash|rb|pl|php|lua)$`)
+
+// fileOperands returns the operands of a line shaped like files: a dot or a
+// slash, no whitespace, not a flag, not a URL and not a script. A flag written
+// as --flag=value contributes its value. The program is not an operand, whether
+// it is the command name or a script handed to an interpreter, so two runs of
+// the same program with different arguments never pair.
+func fileOperands(cmd string) []string {
+	words := shell.Operands(cmd)
+	if words == nil {
+		if fields := strings.Fields(cmd); len(fields) > 1 {
+			words = fields[1:]
+		}
+	}
+	var out []string
+	for _, w := range words {
+		if strings.HasPrefix(w, "-") {
+			_, v, ok := strings.Cut(w, "=")
+			if !ok {
+				continue
 			}
+			w = v
+		}
+		switch {
+		case w == "" || strings.ContainsAny(w, " \t\n"):
+		case strings.HasPrefix(w, "http://"), strings.HasPrefix(w, "https://"), strings.HasPrefix(w, "ftp://"):
+		case reScriptFile.MatchString(w):
+		case strings.ContainsAny(w, "./"):
+			out = append(out, w)
 		}
 	}
+	return out
 }
 
 // rePrerequisiteNear matches the phrasings a tool uses to say that something

dependent_adversarial_test.go

@@ -2,8 +2,11 @@ package main
 
 import (
 	"fmt"
-	kplan "github.com/dcadolph/kibble/internal/plan"
 	"testing"
+
+	"github.com/google/go-cmp/cmp"
+
+	kplan "github.com/dcadolph/kibble/internal/plan"
 )
 
 // TestSuppressionNeedsMoreThanAMention feeds resolveDependentFailures the
@@ -87,3 +90,94 @@ func TestRepeatedSubcommandIsIndependent(t *testing.T) {
 		t.Errorf("status = %s, want FAIL (detail %q)", got, run.Steps[0].Lines[1].Detail)
 	}
 }
+
+// TestSharedOperandAfterLineNotPassed checks the shared-operand rule. A query
+// on a file an earlier line of the same document named and did not pass is
+// blocked behind that line, and nothing weaker than a shared file pairs them.
+func TestSharedOperandAfterLineNotPassed(t *testing.T) {
+	t.Parallel()
+	const query = "Error: no such table: dogs"
+	tests := []struct {
+		Steps      []exampleStep
+		WantStatus []Status
+		WantDetail string
+	}{{ // Test 0: Queries after a blocked insert on the same database are blocked.
+		Steps: []exampleStep{{ID: "b1", Doc: "README.md", Lines: []lineResult{
+			{Cmd: "dbtool insert dogs.db dogs dogs.csv --csv", Status: StatusBlocked},
+			{Cmd: `dbtool dogs.db "select id, name from dogs"`, Status: StatusFail, output: query},
+			{Cmd: `dbtool dogs.db "select * from dogs" --csv`, Status: StatusFail, output: query},
+			{Cmd: `dbtool dogs.db "select * from dogs" --table`, Status: StatusFail, output: query},
+		}}},
+		WantStatus: []Status{StatusBlocked, StatusBlocked, StatusBlocked, StatusBlocked},
+		WantDetail: "names dogs.db, which `dbtool insert dogs.db dogs dogs.csv --csv` also named and did not pass",
+	}, { // Test 1: The same queries after a verified insert stay failures.
+		Steps: []exampleStep{{ID: "b1", Doc: "README.md", Lines: []lineResult{
+			{Cmd: "dbtool insert dogs.db dogs dogs.csv --csv", Status: StatusVerified},
+			{Cmd: `dbtool dogs.db "select id, name from dogs"`, Status: StatusFail, output: query},
+			{Cmd: `dbtool dogs.db "select * from dogs" --csv`, Status: StatusFail, output: query},
+		}}},
+		WantStatus: []Status{StatusVerified, StatusFail, StatusFail},
+	}, { // Test 2: A shared bare word is not a file, so it does not pair.
+		Steps: []exampleStep{{ID: "b1", Doc: "README.md", Lines: []lineResult{
+			{Cmd: "dbtool build release", Status: StatusBlocked},
+			{Cmd: "dbtool test release", Status: StatusFail, output: "error: test failed"},
+		}}},
+		WantStatus: []Status{StatusBlocked, StatusFail},
+	}, { // Test 3: A shared file in a different document does not pair.
+		Steps: []exampleStep{
+			{ID: "b1", Doc: "docs/load.md", Lines: []lineResult{
+				{Cmd: "dbtool insert dogs.db dogs dogs.csv --csv", Status: StatusBlocked},
+			}},
+			{ID: "b2", Doc: "README.md", Lines: []lineResult{
+				{Cmd: `dbtool dogs.db "select * from dogs"`, Status: StatusFail, output: query},
+			}},
+		},
+		WantStatus: []Status{StatusBlocked, StatusFail},
+	}, { // Test 4: A flag's value counts, and a step earlier in the document pairs.
+		Steps: []exampleStep{
+			{ID: "b1", Doc: "README.md", Lines: []lineResult{
+				{Cmd: "dbtool load --file=data/dogs.db", Status: StatusSkipped},
+			}},
+			{ID: "b2", Doc: "README.md", Lines: []lineResult{
+				{Cmd: "dbtool query --db data/dogs.db", Status: StatusFail, output: query},
+			}},
+		},
+		WantStatus: []Status{StatusSkipped, StatusBlocked},
+		WantDetail: "names data/dogs.db, which `dbtool load --file=data/dogs.db` also named and did not pass",
+	}, { // Test 5: A URL and a query string holding a dot are not files.
+		Steps: []exampleStep{{ID: "b1", Doc: "README.md", Lines: []lineResult{
+			{Cmd: `dbtool fetch https://example.com/x "select a.b from t"`, Status: StatusBlocked},
+			{Cmd: `dbtool show https://example.com/x "select a.b from t"`, Status: StatusFail, output: "error"},
+		}}},
+		WantStatus: []Status{StatusBlocked, StatusFail},
+	}, { // Test 6: Two runs of one script with different arguments are separate.
+		Steps: []exampleStep{{ID: "b1", Doc: "README.md", Lines: []lineResult{
+			{Cmd: "python3 scripts/bench.py --engine both", Status: StatusBlocked},
+			{Cmd: "python3 scripts/bench.py --engine one", Status: StatusFail, output: "error"},
+		}}},
+		WantStatus: []Status{StatusBlocked, StatusFail},
+	}}
+	for testNum, test := range tests {
+		t.Run(fmt.Sprintf("test %d", testNum), func(t *testing.T) {
+			t.Parallel()
+			run := &exampleRun{Steps: test.Steps}
+			resolveDependentFailures(run, &kplan.Plan{Binaries: []string{"dbtool"}})
+			var got []Status
+			last := ""
+			for _, s := range run.Steps {
+				for _, l := range s.Lines {
+					got = append(got, l.Status)
+					if l.Reason == ReasonDependsOnSkipped {
+						last = l.Detail
+					}
+				}
+			}
+			if diff := cmp.Diff(test.WantStatus, got); diff != "" {
+				t.Errorf("status mismatch (-want +got):\n%s", diff)
+			}
+			if test.WantDetail != "" && last != test.WantDetail {
+				t.Errorf("detail = %q, want %q", last, test.WantDetail)
+			}
+		})
+	}
+}

docs/DESIGN.md

@@ -211,6 +211,15 @@ directory the document meant. Resetting the working directory at every heading w
 rejected, because it would break every document that changes directory once in setup and
 invent failures.
 
+A `cd` the planner skips, because its target is a placeholder or a name the reader fills
+in, loses the directory for the rest of its section. A document keeps one shell, so every
+later line under the same heading would run in the wrong directory, and each is a `SKIP`
+with the reason `depends-on-skipped` until a `cd` the session can make runs: home, an
+absolute path, or a directory the checkout holds while no earlier `cd` has moved the
+session. A relative `cd` inside the skipped line's own block reads from the lost directory
+and clears nothing. The next heading ends the loss, since a new section starts a new
+procedure, which is the same boundary `inherited-state` reads.
+
 The same holds for the checkout itself. A would-be plain `FAIL` that ran in the
 repository's own checkout, and whose output says the target "already exists" or is
 "already initialized", is `BLOCKED` with the reason `checkout-state`. The collision is
@@ -252,6 +261,15 @@ the document was to write that file, through a redirect, an output flag such as
 verdict is `BLOCKED` with the reason `depends-on-skipped`, naming that line. Only a file no
 earlier line was to write is a `GAP`.
 
+A failing line that names a file-shaped operand, one with a dot or a slash that is not a
+flag or a URL, which an earlier line of the same document also named and which was skipped,
+blocked, or a gap, is `BLOCKED` with the reason `depends-on-skipped`, naming that earlier
+line. A database three queries read after the insert that fills it was blocked is one
+break, not four. An earlier `FAIL` does not pair, so one real break repeated on several
+lines stays several reports, and an operand ending in a script extension such as `.py` or
+`.sh` is the program an interpreter runs rather than data, so two runs of one script never
+pair. Like the other causal downgrades it suppresses the cascade without claiming cause.
+
 A titled fence whose title is a plain relative file path is a file the document writes. The
 planner turns it into a line that writes the block's content, so a later line reading that
 file does not need it, and a path the checkout already holds is never overwritten. A caption
@@ -292,14 +310,31 @@ the classifier cannot tell which. A program the tool tried to run, or one the sh
 could not find, is a `SKIP` for a missing dependency. Git's own report that the
 repository has no history is `BLOCKED`, since the session's repository is one kibble
 made with none. A preset, plugin, or other name the reader is meant to define first, and
-a configuration the tool could not load, are `BLOCKED` as well. Both are resemblances
-rather than names, so neither can ever be a `SKIP` or a pass. The list of nouns is fixed
-on purpose: an unknown flag, argument, option, or command is the tool's own complaint
-and still fails. The refusal counts when the tool names the kind and not the name, as in
-`unknown registry` with nothing after it, and that is `BLOCKED` too. The nouns are the
-same: preset, profile, alias, plugin, group, context, workspace, project, remote,
-registry, template, and theme. A traceback that ends in a missing-file error naming a
-thing is read this way, and one that ends any other way still fails.
+a configuration the tool could not load, use, parse, import, find, or open, are `BLOCKED`
+as well. Both are resemblances rather than names, so neither can ever be a `SKIP` or a
+pass. The list of nouns is fixed on purpose: an unknown flag, argument, option, or command
+is the tool's own complaint and still fails. The refusal counts when the tool names the
+kind and not the name, as in `unknown registry` with nothing after it, and that is
+`BLOCKED` too. The wordings are `unknown <noun>`, `no such <noun>` with or without a name
+after it, `for this <noun>` as in "no dotfile to install for this profile", and `no
+<noun>s are currently installed`. A name the tool reports that the command never passed
+is `BLOCKED` as well, with a detail that says the command never named it, since a name
+the command did not pass came from the tool's default, its environment, or the reader's
+configuration, none of which is the document's claim. The nouns are the same: preset,
+profile, alias, plugin, group, context, workspace, project, remote, registry, template,
+and theme. A tool that tells the reader to define, add, or specify something in or under
+their configuration, as in "define one in your config" or "add a `client` block under an
+`apis:` entry", is pointing at a configuration entry the reader writes first, and is
+`BLOCKED` with the reason `missing-fixture`. So is a tool that asked for a valid path when
+the command passed it a bare word after its subcommand, with no dot and no slash, since a
+name written where a path goes is an alias from the reader's configuration. The same
+message with a file-shaped operand, or with none, still fails. An unquoted path followed
+by "does not exist", as in "generators/php/index.ts does not exist", names a missing file
+the way a quoted one does. A traceback that ends in a missing-file error naming a thing
+is read this way, and one that ends any other way still fails, with two readings of the
+traceback itself: its header counts when other text precedes it on the same line, as a
+progress line without a newline leaves it, and a traceback ending in `EOFError` is a tool
+that waited for an answer on a closed stdin, which is a `SKIP` as interactive.
 
 An operand written as a stand-in name is a placeholder the reader fills in, and the line
 is a `SKIP`. The class is a `my`-prefixed name such as `my-project` or `my_package`, and a
@@ -312,12 +347,28 @@ line runs and does what the document says. A reserved example host is a placehol
 is left to the file rule. A name the repository or an earlier line provides is real, and so
 is a value given to an output flag, since the line writes it. A value written after a
 space cannot be told from an operand without the flag's arity, so it is read as an operand.
+The stand-in class is read on the documented binary's operands and on those of the
+file-reading commands `cd`, `ls`, `tree`, and `cat`, whatever reads the name, and not on
+every command: `git commit -m my-message` is not reading a path. A numbered slot such as
+`{0}` or `{1}` written as its own shell word is a tool's template left for the reader to
+fill, and a placeholder. The same slot inside a quoted argument is the tool's own, as in
+fzf's field bindings, and the line runs.
 
 Check mode is documented behavior. A subcommand such as `check`, `lint`, or `audit`
 exits nonzero when it finds something, and so does a documented tool given a flag that
 says report only: `--fix=false`, `--no-fix`, `--check`, `--dry-run`, or `--diff`. The
-subcommands are `check`, `lint`, `audit`, `vet`, `diff`, `format`, and `fmt`. A nonzero
-exit from either is the tool working. A crash stays a failure. A check-mode flag applies
+subcommands are `check`, `lint`, `audit`, `vet`, `diff`, `format`, and `fmt`, and a
+subcommand the page itself introduces as linting, checking, or validating for problems,
+as in "The `recheck` command lints Markdown files for structure and prose problems", is
+check mode for that document. The inference is by subcommand, never by binary, since a
+linter run over a bad configuration exits nonzero too and blessing the binary would turn
+that into a hollow pass. A nonzero exit from either is the tool working. A crash stays a
+failure, and the classifier now enforces it: a check-mode line whose output holds a
+traceback or a panic falls through to the crash rules. Output the document shows under a
+command as a comment, which the tool then printed, is documented behavior as well: a
+nonzero ordinary exit whose output contains a shown line of at least twelve characters,
+with or without its leading program prefix, is `VERIFIED`, and the match needs the tool to
+have said it, not a resemblance. A check-mode flag applies
 to its own line, while a documented note such as "exits non-zero if it finds any"
 describes the command, so the note reaches every line with the same binary and subcommand
 and the flag does not.

internal/docblock/docblock.go

@@ -477,7 +477,24 @@ var rePlaceholder = regexp.MustCompile(
 // before running it. The planner asks it of an example line and the extractor
 // asks it of an install line, so both read placeholders the same way.
 func HasPlaceholder(s string) bool {
-	return rePlaceholder.MatchString(s)
+	return rePlaceholder.MatchString(s) || hasNumberedSlot(s)
+}
+
+// reNumberedSlot matches a whole shell word that is a numbered template slot
+// such as {0} or {1}, the shape a tool's own command template leaves for the
+// reader to fill.
+var reNumberedSlot = regexp.MustCompile(`^\{[0-9]+\}$`)
+
+// hasNumberedSlot reports whether one of the line's shell words is a numbered
+// slot. It reads parsed words rather than flat text so that a slot inside a
+// quoted argument, which the tool itself fills, is not a placeholder.
+func hasNumberedSlot(s string) bool {
+	for _, w := range shell.ArgWordsOrFields(s) {
+		if reNumberedSlot.MatchString(w) {
+			return true
+		}
+	}
+	return false
 }
 
 // reShownError matches output a doc displays under a command to demonstrate

internal/docblock/docblock_test.go

@@ -86,6 +86,18 @@ func TestHasPlaceholder(t *testing.T) {
 		Want: false,
 	}, { // Test 15: A your-github-username owner is a stand-in.
 		In: `git clone https://github.com/your-github-username/repo`, Want: true,
+	}, { // Test 16: A numbered slot as its own word is a stand-in.
+		In: `tar -cf {1} -C {0} .`, Want: true,
+	}, { // Test 17: Empty braces are find's own token.
+		In: `find . -exec rm {} \;`, Want: false,
+	}, { // Test 18: A brace range is the shell's.
+		In: `echo {1..5}`, Want: false,
+	}, { // Test 19: A repetition count inside a quoted pattern is not a slot.
+		In: `grep -E 'a{2}' f`, Want: false,
+	}, { // Test 20: A numbered slot inside a quoted argument is the tool's own.
+		In: `fzf --delimiter : --nth 3.. --bind 'enter:become(vim {1} +{2})'`, Want: false,
+	}, { // Test 21: A named slot still is.
+		In: `tool {name}`, Want: true,
 	}}
 
 	for testNum, test := range tests {