Skip to content

Ignore files probing optimization: fetch directory entry metadata in batches #3410

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
AlanIWBFT wants to merge 2 commits into
base: master
Choose a base branch
from
+154 −32
Open

Ignore files probing optimization: fetch directory entry metadata in batches #3410

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4e8fb87
Ignore files probing optimization: fetch directory entry metadata in …
AlanIWBFT Mar 10, 2026
385ba0f
Fix fmt
AlanIWBFT May 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
119 changes: 104 additions & 15 deletions crates/ignore/src/dir.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
use std::{
collections::HashMap,
ffi::{OsStr, OsString},
fs::{File, FileType},
fs::{self, File, FileType},
io::{self, BufRead},
path::{Path, PathBuf},
sync::{Arc, RwLock, Weak},
Expand Down Expand Up @@ -150,6 +150,14 @@ struct IgnoreInner {
opts: IgnoreOptions,
}

struct IgnoreFilesFound {
has_ignore: bool,
has_git_ignore: bool,
has_git_dir: bool,
has_jj_dir: bool,
custom_ignore_files: Vec<bool>,
}

impl Ignore {
/// Return the directory path of this matcher.
pub(crate) fn path(&self) -> &Path {
Expand Down Expand Up @@ -254,34 +262,109 @@ impl Ignore {
(Ignore(Arc::new(ig)), err)
}

/// Like add_child, but uses successful read_dir entries to reduce
/// probing when discovering ignore files.
pub(crate) fn add_child_with_entries<P: AsRef<Path>>(
&self,
dir: P,
entries: &[fs::DirEntry],
) -> (Ignore, Option<Error>) {
let files = Self::collect_ignore_files(
entries,
&self.0.custom_ignore_filenames,
);
let (ig, err) = self.add_child_path_with_found_ignore_files(
dir.as_ref(),
Some(&files),
);
(Ignore(Arc::new(ig)), err)
}

/// Like add_child, but takes a full path and returns an IgnoreInner.
fn add_child_path(&self, dir: &Path) -> (IgnoreInner, Option<Error>) {
self.add_child_path_with_found_ignore_files(dir, None)
}

fn collect_ignore_files(
entries: &[fs::DirEntry],
custom_ignore_filenames: &[OsString],
) -> IgnoreFilesFound {
let mut files = IgnoreFilesFound {
has_ignore: false,
has_git_ignore: false,
has_git_dir: false,
has_jj_dir: false,
custom_ignore_files: vec![false; custom_ignore_filenames.len()],
};
for entry in entries {
let file_name = entry.file_name();
if file_name == OsStr::new(".ignore") {
files.has_ignore = true;
} else if file_name == OsStr::new(".gitignore") {
files.has_git_ignore = true;
} else if file_name == OsStr::new(".git") {
files.has_git_dir = true;
} else if file_name == OsStr::new(".jj") {
files.has_jj_dir = true;
}
for (i, name) in custom_ignore_filenames.iter().enumerate() {
if file_name == name.as_os_str() {
files.custom_ignore_files[i] = true;
}
}
}
files
}

fn add_child_path_with_found_ignore_files(
&self,
dir: &Path,
ignore_files_list: Option<&IgnoreFilesFound>,
) -> (IgnoreInner, Option<Error>) {
let check_vcs_dir = self.0.opts.require_git
&& (self.0.opts.git_ignore || self.0.opts.git_exclude);
let git_type = if check_vcs_dir {
let git_type = if check_vcs_dir
&& ignore_files_list.is_none_or(|i| i.has_git_dir)
{
dir.join(".git").metadata().ok().map(|md| md.file_type())
} else {
None
};
let has_git =
check_vcs_dir && (git_type.is_some() || dir.join(".jj").exists());
let has_jj = check_vcs_dir
&& ignore_files_list.is_none_or(|i| i.has_jj_dir)
&& dir.join(".jj").exists();
let has_git = check_vcs_dir && (git_type.is_some() || has_jj);

let mut errs = PartialErrorBuilder::default();
let custom_ig_matcher = if self.0.custom_ignore_filenames.is_empty() {
Gitignore::empty()
} else {
let (m, err) = create_gitignore(
&dir,
&dir,
&self.0.custom_ignore_filenames,
self.0.opts.ignore_case_insensitive,
);
errs.maybe_push(err);
m
let custom_ignore_names: Vec<&OsString> = match ignore_files_list {
None => self.0.custom_ignore_filenames.iter().collect(),
Some(m) => self
.0
.custom_ignore_filenames
.iter()
.zip(m.custom_ignore_files.iter())
.filter_map(|(name, matched)| (*matched).then_some(name))
.collect(),
};
if custom_ignore_names.is_empty() {
Gitignore::empty()
} else {
let (m, err) = create_gitignore(
&dir,
&dir,
&custom_ignore_names,
self.0.opts.ignore_case_insensitive,
);
errs.maybe_push(err);
m
}
};
let ig_matcher = if !self.0.opts.ignore {
Gitignore::empty()
} else {
} else if ignore_files_list.is_none_or(|i| i.has_ignore) {
let (m, err) = create_gitignore(
&dir,
&dir,
Expand All @@ -290,10 +373,12 @@ impl Ignore {
);
errs.maybe_push(err);
m
} else {
Gitignore::empty()
};
let gi_matcher = if !self.0.opts.git_ignore {
Gitignore::empty()
} else {
} else if ignore_files_list.is_none_or(|i| i.has_git_ignore) {
let (m, err) = create_gitignore(
&dir,
&dir,
Expand All @@ -302,11 +387,13 @@ impl Ignore {
);
errs.maybe_push(err);
m
} else {
Gitignore::empty()
};

let gi_exclude_matcher = if !self.0.opts.git_exclude {
Gitignore::empty()
} else {
} else if ignore_files_list.is_none_or(|i| i.has_git_dir) {
match resolve_git_commondir(dir, git_type) {
Ok(git_dir) => {
let (m, err) = create_gitignore(
Expand All @@ -323,6 +410,8 @@ impl Ignore {
Gitignore::empty()
}
}
} else {
Gitignore::empty()
};
let ig = IgnoreInner {
compiled: self.0.compiled.clone(),
Expand Down
67 changes: 50 additions & 17 deletions crates/ignore/src/walk.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1463,6 +1463,12 @@ struct Work {
root_device: Option<u64>,
}

#[derive(Default)]
struct ReadDirResult {
entries: Vec<fs::DirEntry>,
errors: Vec<Error>,
}

impl Work {
/// Returns true if and only if this work item is a directory.
fn is_dir(&self) -> bool {
Expand All @@ -1489,14 +1495,21 @@ impl Work {
err
}

/// Adds ignore rules for this directory without reading its contents.
fn add_ignore(&mut self) {
let (ig, err) = self.ignore.add_child(self.dent.path());
self.ignore = ig;
self.dent.err = err;
}

/// Reads the directory contents of this work item and adds ignore
/// rules for this directory.
///
/// If there was a problem with reading the directory contents, then
/// an error is returned. If there was a problem reading the ignore
/// rules for this directory, then the error is attached to this
/// work item's directory entry.
fn read_dir(&mut self) -> Result<fs::ReadDir, Error> {
fn read_dir(&mut self) -> Result<ReadDirResult, Error> {
let readdir = match fs::read_dir(self.dent.path()) {
Ok(readdir) => readdir,
Err(err) => {
Expand All @@ -1506,10 +1519,24 @@ impl Work {
return Err(err);
}
};
let (ig, err) = self.ignore.add_child(self.dent.path());
// Actually descend into the directory and read its contents
let mut result = ReadDirResult::default();
for entry in readdir {
match entry {
Ok(entry) => result.entries.push(entry),
Err(err) => result.errors.push(
Error::from(err)
.with_path(self.dent.path())
.with_depth(self.dent.depth() + 1),
),
}
}
let (ig, err) = self
.ignore
.add_child_with_entries(self.dent.path(), &result.entries);
self.ignore = ig;
self.dent.err = err;
Ok(readdir)
Ok(result)
}
}

Expand Down Expand Up @@ -1679,8 +1706,15 @@ impl<'s> Worker<'s> {
// have sufficient read permissions to list the directory.
// In that case we still want to provide the closure with a valid
// entry before passing the error value.
let readdir = work.read_dir();
let depth = work.dent.depth();
let readdir = if descend
&& self.max_depth.is_none_or(|m| work.dent.depth() < m)
{
Some(work.read_dir())
} else {
work.add_ignore();
None
};
if should_visit {
let state = self.visitor.visit(Ok(work.dent));
if !state.is_continue() {
Expand All @@ -1691,17 +1725,18 @@ impl<'s> Worker<'s> {
return WalkState::Skip;
}

let readdir = match readdir {
Some(readdir) => readdir,
None => return WalkState::Skip,
};
let readdir = match readdir {
Ok(readdir) => readdir,
Err(err) => {
return self.visitor.visit(Err(err));
}
};

if self.max_depth.map_or(false, |max| depth >= max) {
return WalkState::Skip;
}
for result in readdir {
for result in readdir.entries {
let state = self.generate_work(
&work.ignore,
depth + 1,
Expand All @@ -1712,6 +1747,12 @@ impl<'s> Worker<'s> {
return state;
}
}
for err in readdir.errors {
let state = self.visitor.visit(Err(err));
if state.is_quit() {
return state;
}
}
WalkState::Continue
}

Expand All @@ -1733,16 +1774,8 @@ impl<'s> Worker<'s> {
ig: &Ignore,
depth: usize,
root_device: Option<u64>,
result: Result<fs::DirEntry, io::Error>,
fs_dent: fs::DirEntry,
) -> WalkState {
let fs_dent = match result {
Ok(fs_dent) => fs_dent,
Err(err) => {
return self
.visitor
.visit(Err(Error::from(err).with_depth(depth)));
}
};
let mut dent = match DirEntryRaw::from_entry(depth, &fs_dent) {
Ok(dent) => DirEntry::new_raw(dent, None),
Err(err) => {
Expand Down
Loading