Skip to content

fix(grep): skip binary and non-UTF-8 input instead of aborting #124

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
so0k wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix(grep): skip binary and non-UTF-8 input instead of aborting #124

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
140 changes: 129 additions & 11 deletions src/commands/grep.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
use std::borrow::Cow;

use crate::os;
use crate::prelude::*;

Expand All @@ -21,6 +23,9 @@ Options:
-H, --with-filename print filename with matches
-h, --no-filename suppress filename prefix
-o, --only-matching show only the matching part
-a, --text process a binary file as if it were text
-I ignore binary files (treat as non-matching)
--binary-files=TYPE binary, text, or without-match
-m, --max-count NUM stop after NUM matches per file
-A, --after-context NUM print NUM lines after match
-B, --before-context NUM print NUM lines before match
Expand All @@ -29,6 +34,17 @@ Options:
--exclude=GLOB skip files matching GLOB
--exclude-dir=DIR skip directories matching DIR";

/// How to treat input that looks binary (NUL byte or invalid UTF-8).
#[derive(Clone, Copy, PartialEq, Eq)]
enum BinaryFiles {
/// Report "binary file matches" instead of printing the lines.
Binary,
/// Print every line, decoding invalid bytes lossily.
Text,
/// Treat files containing NUL as if they had no matches.
WithoutMatch,
}

struct Opts {
patterns: Vec<String>,
files: Vec<String>,
Expand All @@ -46,6 +62,7 @@ struct Opts {
with_filename: Option<bool>,
only_matching: bool,
max_count: Option<u64>,
binary_files: BinaryFiles,
after_context: usize,
before_context: usize,
include: Vec<String>,
Expand All @@ -71,6 +88,7 @@ fn parse_args(args: &[String]) -> Result<Option<Opts>, Box<dyn std::error::Error
with_filename: None,
only_matching: false,
max_count: None,
binary_files: BinaryFiles::Binary,
after_context: 0,
before_context: 0,
include: Vec::new(),
Expand All @@ -97,6 +115,20 @@ fn parse_args(args: &[String]) -> Result<Option<Opts>, Box<dyn std::error::Error
Short('h') | Long("no-filename") => opts.with_filename = Some(false),
Short('o') | Long("only-matching") => opts.only_matching = true,
Short('m') | Long("max-count") => opts.max_count = Some(parser.value()?.parse()?),
Short('a') | Long("text") => opts.binary_files = BinaryFiles::Text,
Short('I') => opts.binary_files = BinaryFiles::WithoutMatch,
Long("binary-files") => {
opts.binary_files = match parser.value()?.string()?.as_str() {
"binary" => BinaryFiles::Binary,
"text" => BinaryFiles::Text,
"without-match" => BinaryFiles::WithoutMatch,
other => {
return Err(
format!("invalid argument '{other}' for '--binary-files'").into()
);
}
};
}
Short('A') | Long("after-context") => opts.after_context = parser.value()?.parse()?,
Short('B') | Long("before-context") => opts.before_context = parser.value()?.parse()?,
Short('C') | Long("context") => {
Expand Down Expand Up @@ -216,13 +248,21 @@ async fn grep_reader<R: tokio::io::AsyncRead + Unpin + Send>(
re: &regex::Regex,
opts: &Opts,
prefix: &str,
name: &str,
w: &mut os::FdWriter,
ew: &mut os::FdWriter,
) -> Result<bool, Box<dyn std::error::Error + Send + Sync>> {
let mut buf_reader = BufReader::new(reader);
let mut line = String::new();
let mut line: Vec<u8> = Vec::new();
let mut lineno: u64 = 0;
let mut match_count: u64 = 0;
let mut found = false;
let detect_binary = opts.binary_files != BinaryFiles::Text;
// A NUL in the first buffer marks the whole input binary; a later NUL hides only its line.
let file_binary = detect_binary && buf_reader.fill_buf().await?.contains(&0);
// A matched or context line was hidden as binary, so report it on stderr.
let mut binary_matched = false;
let summary_only = opts.quiet || opts.list || opts.list_non_matching || opts.count;

let use_context = opts.before_context > 0 || opts.after_context > 0;
// Ring buffer for before-context
Expand All @@ -232,21 +272,42 @@ async fn grep_reader<R: tokio::io::AsyncRead + Unpin + Send>(
let mut after_remaining: usize = 0;
// Whether we need a "--" separator before the next context group
let mut need_sep = false;
// Whether any match or context line has been written yet.
let mut printed_any = false;

// A binary file is treated as non-matching under -I.
let skip_file = file_binary && opts.binary_files == BinaryFiles::WithoutMatch;
// Line number of the last binary line hidden from context (0 = none).
let mut last_binary_lineno: u64 = 0;

loop {
if skip_file {
break;
}
line.clear();
if buf_reader.read_line(&mut line).await? == 0 {
if buf_reader.read_until(b'\n', &mut line).await? == 0 {
break;
}
lineno += 1;
let text = line.trim_end_matches('\n').trim_end_matches('\r');
let mut raw = line.as_slice();
raw = raw.strip_suffix(b"\n").unwrap_or(raw);
raw = raw.strip_suffix(b"\r").unwrap_or(raw);
// Invalid UTF-8 is matched lossily but never printed unless -a.
let (text, is_binary) = match std::str::from_utf8(raw) {
Ok(s) => (
Cow::Borrowed(s),
file_binary || (detect_binary && s.contains('\0')),
),
Err(_) => (String::from_utf8_lossy(raw), detect_binary),
};
let text = text.as_ref();
let matched = re.is_match(text) ^ opts.invert;

if matched {
found = true;
match_count += 1;

if opts.quiet || opts.list || opts.list_non_matching || opts.count {
if summary_only {
if let Some(max) = opts.max_count
&& match_count >= max
{
Expand All @@ -255,12 +316,34 @@ async fn grep_reader<R: tokio::io::AsyncRead + Unpin + Send>(
continue;
}

if is_binary {
binary_matched = true;
if use_context {
// The hidden match ends any open context and breaks the group.
after_remaining = 0;
before_buf.clear();
need_sep = printed_any;
last_binary_lineno = lineno;
}
if file_binary || opts.max_count.is_some_and(|max| match_count >= max) {
break;
}
continue;
}

if use_context {
// A hidden binary line still inside the before window.
let binary_in_window = last_binary_lineno > 0
&& lineno - last_binary_lineno <= opts.before_context as u64;
// Print separator between context groups
if (opts.before_context == 0 || !before_buf.is_empty()) && need_sep {
if (opts.before_context == 0 || !before_buf.is_empty() || binary_in_window)
&& need_sep
{
wprintln!(w, "--")?;
}
need_sep = false;
binary_matched |= binary_in_window;
last_binary_lineno = 0;
// Flush before-context buffer
for (bno, btext) in before_buf.drain(..) {
if !prefix.is_empty() {
Expand All @@ -274,6 +357,7 @@ async fn grep_reader<R: tokio::io::AsyncRead + Unpin + Send>(
after_remaining = opts.after_context;
}

printed_any = true;
if opts.only_matching && !opts.invert {
for m in re.find_iter(text) {
if !prefix.is_empty() {
Expand All @@ -299,9 +383,25 @@ async fn grep_reader<R: tokio::io::AsyncRead + Unpin + Send>(
{
break;
}
} else if is_binary {
// Binary lines are never shown, but they still occupy a context slot.
if use_context {
if found && after_remaining > 0 {
after_remaining -= 1;
// -o prints no context, so the hidden line is never in play.
binary_matched |= !opts.only_matching;
if after_remaining == 0 {
need_sep = true;
}
} else {
before_buf.clear();
last_binary_lineno = lineno;
}
}
} else if use_context && found && after_remaining > 0 {
// Print after-context line
after_remaining -= 1;
printed_any = true;
if !prefix.is_empty() {
wprint!(w, "{}-", prefix)?;
}
Expand Down Expand Up @@ -336,6 +436,9 @@ async fn grep_reader<R: tokio::io::AsyncRead + Unpin + Send>(
if opts.list_non_matching && !found {
wprintln!(w, "{}", prefix)?;
}
if binary_matched && !summary_only && opts.binary_files == BinaryFiles::Binary {
wprintln!(ew, "grep: {}: binary file matches", name)?;
}

Ok(found)
}
Expand Down Expand Up @@ -377,12 +480,14 @@ async fn cmd_grep(os: &dyn Kernel, args: &[String]) -> CommandResult {
let show_name = opts.with_filename.unwrap_or(multi);

let mut w = io::stdout()?;
// stderr can only be taken once per process, so share one writer across files.
let mut ew = io::stderr()?;
let mut any_match = false;

if files.is_empty() {
// Read from stdin
let reader = io::stdin()?;
if grep_reader(reader, &re, &opts, "", &mut w).await? {
if grep_reader(reader, &re, &opts, "", "(standard input)", &mut w, &mut ew).await? {
any_match = true;
}
} else {
Expand All @@ -391,18 +496,31 @@ async fn cmd_grep(os: &dyn Kernel, args: &[String]) -> CommandResult {
Ok(fd) => fd,
Err(e) => {
if !opts.quiet {
let mut ew = io::stderr()?;
wprintln!(ew, "grep: {}: {}", path, e)?;
}
continue;
}
};
let reader = io::take_reader(fd)?;
let prefix = if show_name { path.as_str() } else { "" };
if grep_reader(reader, &re, &opts, prefix, &mut w).await? {
any_match = true;
if opts.quiet {
return Ok(0);
match grep_reader(reader, &re, &opts, prefix, path, &mut w, &mut ew).await {
Ok(true) => {
any_match = true;
if opts.quiet {
return Ok(0);
}
}
Ok(false) => {}
Err(e) => {
// A closed stdout must stop grep, not repeat per file.
if e.downcast_ref::<std::io::Error>()
.is_some_and(|io| io.kind() == std::io::ErrorKind::BrokenPipe)
{
return Err(e);
}
if !opts.quiet {
wprintln!(ew, "grep: {}: {}", path, e)?;
}
}
}
}
Expand Down
Loading