diff --git a/src/commands/grep.rs b/src/commands/grep.rs index ba3b382..8f12202 100644 --- a/src/commands/grep.rs +++ b/src/commands/grep.rs @@ -1,3 +1,5 @@ +use std::borrow::Cow; + use crate::os; use crate::prelude::*; @@ -21,6 +23,9 @@ Options: -H, --with-filename print filename with matches -h, --no-filename suppress filename prefix -o, --only-matching show only the matching part + -a, --text process a binary file as if it were text + -I ignore binary files (treat as non-matching) + --binary-files=TYPE binary, text, or without-match -m, --max-count NUM stop after NUM matches per file -A, --after-context NUM print NUM lines after match -B, --before-context NUM print NUM lines before match @@ -29,6 +34,17 @@ Options: --exclude=GLOB skip files matching GLOB --exclude-dir=DIR skip directories matching DIR"; +/// How to treat input that looks binary (NUL byte or invalid UTF-8). +#[derive(Clone, Copy, PartialEq, Eq)] +enum BinaryFiles { + /// Report "binary file matches" instead of printing the lines. + Binary, + /// Print every line, decoding invalid bytes lossily. + Text, + /// Treat files containing NUL as if they had no matches. + WithoutMatch, +} + struct Opts { patterns: Vec, files: Vec, @@ -46,6 +62,7 @@ struct Opts { with_filename: Option, only_matching: bool, max_count: Option, + binary_files: BinaryFiles, after_context: usize, before_context: usize, include: Vec, @@ -71,6 +88,7 @@ fn parse_args(args: &[String]) -> Result, Box Result, Box opts.with_filename = Some(false), Short('o') | Long("only-matching") => opts.only_matching = true, Short('m') | Long("max-count") => opts.max_count = Some(parser.value()?.parse()?), + Short('a') | Long("text") => opts.binary_files = BinaryFiles::Text, + Short('I') => opts.binary_files = BinaryFiles::WithoutMatch, + Long("binary-files") => { + opts.binary_files = match parser.value()?.string()?.as_str() { + "binary" => BinaryFiles::Binary, + "text" => BinaryFiles::Text, + "without-match" => BinaryFiles::WithoutMatch, + other => { + return Err( + format!("invalid argument '{other}' for '--binary-files'").into() + ); + } + }; + } Short('A') | Long("after-context") => opts.after_context = parser.value()?.parse()?, Short('B') | Long("before-context") => opts.before_context = parser.value()?.parse()?, Short('C') | Long("context") => { @@ -216,13 +248,21 @@ async fn grep_reader( re: ®ex::Regex, opts: &Opts, prefix: &str, + name: &str, w: &mut os::FdWriter, + ew: &mut os::FdWriter, ) -> Result> { let mut buf_reader = BufReader::new(reader); - let mut line = String::new(); + let mut line: Vec = Vec::new(); let mut lineno: u64 = 0; let mut match_count: u64 = 0; let mut found = false; + let detect_binary = opts.binary_files != BinaryFiles::Text; + // A NUL in the first buffer marks the whole input binary; a later NUL hides only its line. + let file_binary = detect_binary && buf_reader.fill_buf().await?.contains(&0); + // A matched or context line was hidden as binary, so report it on stderr. + let mut binary_matched = false; + let summary_only = opts.quiet || opts.list || opts.list_non_matching || opts.count; let use_context = opts.before_context > 0 || opts.after_context > 0; // Ring buffer for before-context @@ -232,21 +272,42 @@ async fn grep_reader( let mut after_remaining: usize = 0; // Whether we need a "--" separator before the next context group let mut need_sep = false; + // Whether any match or context line has been written yet. + let mut printed_any = false; + + // A binary file is treated as non-matching under -I. + let skip_file = file_binary && opts.binary_files == BinaryFiles::WithoutMatch; + // Line number of the last binary line hidden from context (0 = none). + let mut last_binary_lineno: u64 = 0; loop { + if skip_file { + break; + } line.clear(); - if buf_reader.read_line(&mut line).await? == 0 { + if buf_reader.read_until(b'\n', &mut line).await? == 0 { break; } lineno += 1; - let text = line.trim_end_matches('\n').trim_end_matches('\r'); + let mut raw = line.as_slice(); + raw = raw.strip_suffix(b"\n").unwrap_or(raw); + raw = raw.strip_suffix(b"\r").unwrap_or(raw); + // Invalid UTF-8 is matched lossily but never printed unless -a. + let (text, is_binary) = match std::str::from_utf8(raw) { + Ok(s) => ( + Cow::Borrowed(s), + file_binary || (detect_binary && s.contains('\0')), + ), + Err(_) => (String::from_utf8_lossy(raw), detect_binary), + }; + let text = text.as_ref(); let matched = re.is_match(text) ^ opts.invert; if matched { found = true; match_count += 1; - if opts.quiet || opts.list || opts.list_non_matching || opts.count { + if summary_only { if let Some(max) = opts.max_count && match_count >= max { @@ -255,12 +316,34 @@ async fn grep_reader( continue; } + if is_binary { + binary_matched = true; + if use_context { + // The hidden match ends any open context and breaks the group. + after_remaining = 0; + before_buf.clear(); + need_sep = printed_any; + last_binary_lineno = lineno; + } + if file_binary || opts.max_count.is_some_and(|max| match_count >= max) { + break; + } + continue; + } + if use_context { + // A hidden binary line still inside the before window. + let binary_in_window = last_binary_lineno > 0 + && lineno - last_binary_lineno <= opts.before_context as u64; // Print separator between context groups - if (opts.before_context == 0 || !before_buf.is_empty()) && need_sep { + if (opts.before_context == 0 || !before_buf.is_empty() || binary_in_window) + && need_sep + { wprintln!(w, "--")?; } need_sep = false; + binary_matched |= binary_in_window; + last_binary_lineno = 0; // Flush before-context buffer for (bno, btext) in before_buf.drain(..) { if !prefix.is_empty() { @@ -274,6 +357,7 @@ async fn grep_reader( after_remaining = opts.after_context; } + printed_any = true; if opts.only_matching && !opts.invert { for m in re.find_iter(text) { if !prefix.is_empty() { @@ -299,9 +383,25 @@ async fn grep_reader( { break; } + } else if is_binary { + // Binary lines are never shown, but they still occupy a context slot. + if use_context { + if found && after_remaining > 0 { + after_remaining -= 1; + // -o prints no context, so the hidden line is never in play. + binary_matched |= !opts.only_matching; + if after_remaining == 0 { + need_sep = true; + } + } else { + before_buf.clear(); + last_binary_lineno = lineno; + } + } } else if use_context && found && after_remaining > 0 { // Print after-context line after_remaining -= 1; + printed_any = true; if !prefix.is_empty() { wprint!(w, "{}-", prefix)?; } @@ -336,6 +436,9 @@ async fn grep_reader( if opts.list_non_matching && !found { wprintln!(w, "{}", prefix)?; } + if binary_matched && !summary_only && opts.binary_files == BinaryFiles::Binary { + wprintln!(ew, "grep: {}: binary file matches", name)?; + } Ok(found) } @@ -377,12 +480,14 @@ async fn cmd_grep(os: &dyn Kernel, args: &[String]) -> CommandResult { let show_name = opts.with_filename.unwrap_or(multi); let mut w = io::stdout()?; + // stderr can only be taken once per process, so share one writer across files. + let mut ew = io::stderr()?; let mut any_match = false; if files.is_empty() { // Read from stdin let reader = io::stdin()?; - if grep_reader(reader, &re, &opts, "", &mut w).await? { + if grep_reader(reader, &re, &opts, "", "(standard input)", &mut w, &mut ew).await? { any_match = true; } } else { @@ -391,7 +496,6 @@ async fn cmd_grep(os: &dyn Kernel, args: &[String]) -> CommandResult { Ok(fd) => fd, Err(e) => { if !opts.quiet { - let mut ew = io::stderr()?; wprintln!(ew, "grep: {}: {}", path, e)?; } continue; @@ -399,10 +503,24 @@ async fn cmd_grep(os: &dyn Kernel, args: &[String]) -> CommandResult { }; let reader = io::take_reader(fd)?; let prefix = if show_name { path.as_str() } else { "" }; - if grep_reader(reader, &re, &opts, prefix, &mut w).await? { - any_match = true; - if opts.quiet { - return Ok(0); + match grep_reader(reader, &re, &opts, prefix, path, &mut w, &mut ew).await { + Ok(true) => { + any_match = true; + if opts.quiet { + return Ok(0); + } + } + Ok(false) => {} + Err(e) => { + // A closed stdout must stop grep, not repeat per file. + if e.downcast_ref::() + .is_some_and(|io| io.kind() == std::io::ErrorKind::BrokenPipe) + { + return Err(e); + } + if !opts.quiet { + wprintln!(ew, "grep: {}: {}", path, e)?; + } } } } diff --git a/tests/shell_integration.rs b/tests/shell_integration.rs index 2547018..40ebb92 100644 --- a/tests/shell_integration.rs +++ b/tests/shell_integration.rs @@ -1982,6 +1982,236 @@ expect!( "/tmp/gi.txt:yes" ); +// ── grep binary / non-UTF-8 input ─────────────────────────────────── + +/// Runs `cmd` in a shell with a host dir of raw-byte fixtures bound at /g. +fn grep_binary_run(tag: &str, cmd: &str) -> strands_shell::Output { + let (rt, local) = rt(); + rt.block_on(local.run_until(async { + let dir = std::env::temp_dir().join(format!("lsh_grep_bin_{tag}")); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("a.txt"), b"needle\n").unwrap(); + std::fs::write(dir.join("b.txt"), b"needle b\n").unwrap(); + std::fs::write(dir.join("bin.dat"), b"\0\xffneedle\n").unwrap(); + std::fs::write(dir.join("c.txt"), b"needle c\n").unwrap(); + std::fs::write( + dir.join("mid.txt"), + b"x\nneedle ok\n\xffneedle bad\nneedle after\n", + ) + .unwrap(); + std::fs::write( + dir.join("ctx.txt"), + b"before\n\xffbad ctx\nneedle hit\n\xffafter bad\nplain after\n", + ) + .unwrap(); + std::fs::write( + dir.join("ctxc.txt"), + b"needle a\n\xffneedle b\nafter1\nneedle c\nafter2\nafter3\n", + ) + .unwrap(); + std::fs::write(dir.join("far.txt"), b"\xffbad\nl2\nl3\nneedle\n").unwrap(); + std::fs::write( + dir.join("sep.txt"), + b"needle A\nl2\nl3\n\xffbin\nneedle B\n", + ) + .unwrap(); + for n in 1..=3 { + let big: String = (0..5000).map(|i| format!("needle {i}\n")).collect(); + std::fs::write(dir.join(format!("big{n}.txt")), big).unwrap(); + } + std::fs::write(dir.join("bin2.dat"), b"\xffneedle\nneedle\n").unwrap(); + std::fs::write(dir.join("nomatch.dat"), b"\0\xffzzz\n").unwrap(); + let mut shell = Shell::builder() + .bind_direct(dir.to_str().unwrap(), "/g") + .build() + .unwrap(); + let out = shell.run(cmd).await; + let _ = std::fs::remove_dir_all(&dir); + out + })) +} + +#[test] +fn grep_binary_file_note() { + let out = grep_binary_run("note", "grep needle /g/bin.dat"); + assert_eq!(out.stdout, ""); + assert_eq!(out.stderr, "grep: /g/bin.dat: binary file matches\n"); + assert_eq!(out.status, 0); +} + +#[test] +fn grep_recursive_continues_past_binary() { + let out = grep_binary_run("rec", "cd /g; grep -rn needle ."); + assert!(out.stdout.contains("a.txt:1:needle"), "{:?}", out.stdout); + assert!(out.stdout.contains("b.txt:1:needle b"), "{:?}", out.stdout); + assert!(out.stdout.contains("c.txt:1:needle c"), "{:?}", out.stdout); + assert!( + out.stdout.contains("mid.txt:2:needle ok"), + "{:?}", + out.stdout + ); + assert!(!out.stdout.contains("bin.dat"), "{:?}", out.stdout); + assert!( + out.stderr.contains("grep: bin.dat: binary file matches"), + "{:?}", + out.stderr + ); + assert!( + out.stderr.contains("grep: mid.txt: binary file matches"), + "{:?}", + out.stderr + ); + assert_eq!(out.status, 0); +} + +#[test] +fn grep_invalid_utf8_line_suppressed() { + let out = grep_binary_run("mid", "grep needle /g/mid.txt"); + assert_eq!(out.stdout, "needle ok\nneedle after\n"); + assert_eq!(out.stderr, "grep: /g/mid.txt: binary file matches\n"); + assert_eq!(out.status, 0); +} + +#[test] +fn grep_only_matching_skips_binary_line() { + let out = grep_binary_run("o", "grep -o needle /g/mid.txt"); + assert_eq!(out.stdout, "needle\nneedle\n"); +} + +#[test] +fn grep_i_skips_binary_file() { + let out = grep_binary_run("bi", "grep -I needle /g/bin.dat"); + assert_eq!(out.stdout, ""); + assert_eq!(out.stderr, ""); + assert_eq!(out.status, 1); +} + +#[test] +fn grep_text_flag_prints_binary_lines() { + let out = grep_binary_run("ba", "grep -a needle /g/mid.txt"); + assert!(out.stdout.contains("needle ok"), "{:?}", out.stdout); + assert!( + out.stdout.contains("\u{FFFD}needle bad"), + "{:?}", + out.stdout + ); + assert_eq!(out.stderr, ""); + assert_eq!(out.status, 0); +} + +#[test] +fn grep_binary_files_option() { + let out = grep_binary_run("bf1", "grep --binary-files=without-match needle /g/bin.dat"); + assert_eq!(out.status, 1); + let out = grep_binary_run("bf2", "grep --binary-files=text needle /g/mid.txt"); + assert!(out.stdout.contains("needle bad"), "{:?}", out.stdout); + let out = grep_binary_run("bf3", "grep --binary-files=bogus needle /g/a.txt"); + assert_ne!(out.status, 0); + assert!( + out.stderr.contains("grep: invalid argument 'bogus'"), + "{:?}", + out.stderr + ); + assert!(!out.stderr.contains("grep: grep:"), "{:?}", out.stderr); +} + +#[test] +fn grep_binary_line_occupies_context_slot() { + let out = grep_binary_run("ctxa", "grep -n -A1 needle /g/ctx.txt"); + assert_eq!(out.stdout, "3:needle hit\n"); + assert!( + out.stderr.contains("binary file matches"), + "{:?}", + out.stderr + ); + let out = grep_binary_run("ctxb", "grep -n -B1 needle /g/ctx.txt"); + assert_eq!(out.stdout, "3:needle hit\n"); + assert!( + out.stderr.contains("binary file matches"), + "{:?}", + out.stderr + ); +} + +#[test] +fn grep_matched_binary_line_resets_context() { + let out = grep_binary_run("ctxc1", "grep -n -A1 needle /g/ctxc.txt"); + assert_eq!(out.stdout, "1:needle a\n--\n4:needle c\n5-after2\n"); + let out = grep_binary_run("ctxc2", "grep -n -A2 needle /g/ctxc.txt"); + assert_eq!( + out.stdout, + "1:needle a\n--\n4:needle c\n5-after2\n6-after3\n" + ); +} + +#[test] +fn grep_distant_binary_line_gives_no_note() { + let out = grep_binary_run("far", "grep -n -B1 needle /g/far.txt"); + assert_eq!(out.stdout, "3-l3\n4:needle\n"); + assert_eq!(out.stderr, ""); +} + +#[test] +fn grep_dropped_binary_line_keeps_group_separator() { + let out = grep_binary_run("sepb", "grep -n -B1 needle /g/sep.txt"); + assert_eq!(out.stdout, "1:needle A\n--\n5:needle B\n"); + let out = grep_binary_run("sepc", "grep -n -C1 needle /g/sep.txt"); + assert_eq!(out.stdout, "1:needle A\n2-l2\n--\n5:needle B\n"); +} + +#[test] +fn grep_without_match_is_silent_for_invalid_utf8() { + let out = grep_binary_run("iq", "grep -I needle /g/mid.txt"); + assert_eq!(out.stdout, "needle ok\nneedle after\n"); + assert_eq!(out.stderr, ""); +} + +#[test] +fn grep_count_and_list_on_binary() { + let out = grep_binary_run("cl", "grep -c needle /g/bin.dat /g/mid.txt"); + assert_eq!(out.stdout, "/g/bin.dat:1\n/g/mid.txt:3\n"); + assert_eq!(out.stderr, ""); + let out = grep_binary_run("cl2", "grep -l needle /g/bin.dat /g/mid.txt"); + assert_eq!(out.stdout, "/g/bin.dat\n/g/mid.txt\n"); + assert_eq!(out.stderr, ""); + let out = grep_binary_run("cl3", "grep -q needle /g/bin.dat"); + assert_eq!(out.status, 0); + assert_eq!(out.stderr, ""); +} + +#[test] +fn grep_stdin_binary_note() { + let out = grep_binary_run("stdin", "cat /g/bin.dat | grep needle"); + assert_eq!(out.stdout, ""); + assert_eq!(out.stderr, "grep: (standard input): binary file matches\n"); + assert_eq!(out.status, 0); +} + +#[test] +fn grep_broken_pipe_stops_after_first_file() { + let out = grep_binary_run( + "bp", + "grep needle /g/big1.txt /g/big2.txt /g/big3.txt | head -n 1", + ); + assert_eq!(out.stdout, "/g/big1.txt:needle 0\n"); + assert!(!out.stderr.contains("/g/big"), "stderr: {}", out.stderr); +} + +#[test] +fn grep_no_leading_separator_before_hidden_binary_match() { + let out = grep_binary_run("sep0", "grep -A1 needle /g/bin2.dat"); + assert_eq!(out.stdout, "needle\n"); +} + +#[test] +fn grep_binary_no_match_quiet_exit_1() { + let out = grep_binary_run("nm", "grep needle /g/nomatch.dat"); + assert_eq!(out.stdout, ""); + assert_eq!(out.stderr, ""); + assert_eq!(out.status, 1); +} + // ── jq coverage ───────────────────────────────────────────────────── expect!(jq_identity, "echo '{\"a\":1}' | jq '.'", "{\n \"a\": 1\n}");