windows: fix OneDrive traversals

This commit fixes a bug on Windows where directory traversals were
completely broken when attempting to scan OneDrive directories that use
the "file on demand" strategy.

The specific problem was that Rust's standard library treats OneDrive
directories as reparse points instead of directories, which causes
methods like `FileType::is_file` and `FileType::is_dir` to always return
false, even when retrieved via methods like `metadata` that purport to
follow symbolic links.

We fix this by peppering our code with checks on the underlying file
attributes exposed by Windows. We consider an entry a directory if and
only if the directory bit is set on the attributes. We are careful to
make sure that the code remains the same on non-Windows platforms.

Note that we also bump the dependency on `walkdir`, which contains a
similar fix for its traversals.

This bug is recorded upstream:
rust-lang/rust#46484

Upstream also has a pending PR:
rust-lang/rust#47956

Fixes #705

Author: BurntSushi

Cargo.lock

@@ -51,6 +51,10 @@ same-file = "1"
 termcolor = { version = "0.3.3", path = "termcolor" }
 globset = { version = "0.2.1", path = "globset" }
 
+[target.'cfg(windows)'.dependencies.winapi]
+version = "0.3"
+features = ["std", "winnt"]
+
 [build-dependencies]
 clap = "2.26"
 lazy_static = "1"

Cargo.toml

@@ -28,6 +28,10 @@ same-file = "1"
 thread_local = "0.3.2"
 walkdir = "2"
 
+[target.'cfg(windows)'.dependencies.winapi]
+version = "0.3"
+features = ["std", "winnt"]
+
 [dev-dependencies]
 tempdir = "0.3.5"
 

ignore/Cargo.toml

@@ -59,6 +59,8 @@ extern crate same_file;
 extern crate tempdir;
 extern crate thread_local;
 extern crate walkdir;
+#[cfg(windows)]
+extern crate winapi;
 
 use std::error;
 use std::fmt;

ignore/src/lib.rs

@@ -89,6 +89,11 @@ impl DirEntry {
         self.err.as_ref()
     }
 
+    /// Returns true if and only if this entry points to a directory.
+    fn is_dir(&self) -> bool {
+        self.dent.is_dir()
+    }
+
     fn new_stdin() -> DirEntry {
         DirEntry {
             dent: DirEntryInner::Stdin,
@@ -208,6 +213,24 @@ impl DirEntryInner {
             Raw(ref x) => Some(x.ino()),
         }
     }
+
+    /// Returns true if and only if this entry points to a directory.
+    ///
+    /// This works around a bug in Rust's standard library:
+    /// https://github.com/rust-lang/rust/issues/46484
+    #[cfg(windows)]
+    fn is_dir(&self) -> bool {
+        self.metadata().map(|md| metadata_is_dir(&md)).unwrap_or(false)
+    }
+
+    /// Returns true if and only if this entry points to a directory.
+    ///
+    /// This works around a bug in Rust's standard library:
+    /// https://github.com/rust-lang/rust/issues/46484
+    #[cfg(not(windows))]
+    fn is_dir(&self) -> bool {
+        self.file_type().map(|ft| ft.is_dir()).unwrap_or(false)
+    }
 }
 
 /// DirEntryRaw is essentially copied from the walkdir crate so that we can
@@ -456,7 +479,7 @@ impl WalkBuilder {
                 (p.to_path_buf(), None)
             } else {
                 let mut wd = WalkDir::new(p);
-                wd = wd.follow_links(follow_links || p.is_file());
+                wd = wd.follow_links(follow_links || path_is_file(p));
                 if let Some(max_depth) = max_depth {
                     wd = wd.max_depth(max_depth);
                 }
@@ -728,7 +751,7 @@ impl Walk {
             return false;
         }
 
-        let is_dir = ent.file_type().is_dir();
+        let is_dir = walkdir_entry_is_dir(ent);
         let max_size = self.max_filesize;
         let should_skip_path = skip_path(&self.ig, ent.path(), is_dir);
         let should_skip_filesize = if !is_dir && max_size.is_some() {
@@ -757,7 +780,7 @@ impl Iterator for Walk {
                         }
                         Some((path, Some(it))) => {
                             self.it = Some(it);
-                            if self.parents && path.is_dir() {
+                            if self.parents && path_is_dir(&path) {
                                 let (ig, err) = self.ig_root.add_parents(path);
                                 self.ig = ig;
                                 if let Some(err) = err {
@@ -847,7 +870,7 @@ impl Iterator for WalkEventIter {
             None => None,
             Some(Err(err)) => Some(Err(err)),
             Some(Ok(dent)) => {
-                if dent.file_type().is_dir() {
+                if walkdir_entry_is_dir(&dent) {
                     self.depth += 1;
                     Some(Ok(WalkEvent::Dir(dent)))
                 } else {
@@ -1000,7 +1023,7 @@ struct Work {
 impl Work {
     /// Returns true if and only if this work item is a directory.
     fn is_dir(&self) -> bool {
-        self.dent.file_type().map_or(false, |t| t.is_dir())
+        self.dent.is_dir()
     }
 
     /// Adds ignore rules for parent directories.
@@ -1174,13 +1197,13 @@ impl Worker {
                     return (self.f)(Err(err));
                 }
             };
-            if dent.file_type().map_or(false, |ft| ft.is_dir()) {
+            if dent.is_dir() {
                 if let Err(err) = check_symlink_loop(ig, dent.path(), depth) {
                     return (self.f)(Err(err));
                 }
             }
         }
-        let is_dir = dent.file_type().map_or(false, |ft| ft.is_dir());
+        let is_dir = dent.is_dir();
         let max_size = self.max_filesize;
         let should_skip_path = skip_path(ig, dent.path(), is_dir);
         let should_skip_filesize = if !is_dir && max_size.is_some() {
@@ -1376,6 +1399,62 @@ fn skip_path(ig: &Ignore, path: &Path, is_dir: bool) -> bool {
     }
 }
 
+/// Returns true if and only if this path points to a directory.
+///
+/// This works around a bug in Rust's standard library:
+/// https://github.com/rust-lang/rust/issues/46484
+#[cfg(windows)]
+fn path_is_dir(path: &Path) -> bool {
+    fs::metadata(path).map(|md| metadata_is_dir(&md)).unwrap_or(false)
+}
+
+/// Returns true if and only if this entry points to a directory.
+#[cfg(not(windows))]
+fn path_is_dir(path: &Path) -> bool {
+    path.is_dir()
+}
+
+/// Returns true if and only if this path points to a file.
+///
+/// This works around a bug in Rust's standard library:
+/// https://github.com/rust-lang/rust/issues/46484
+#[cfg(windows)]
+fn path_is_file(path: &Path) -> bool {
+    !path_is_dir(path)
+}
+
+/// Returns true if and only if this entry points to a directory.
+#[cfg(not(windows))]
+fn path_is_file(path: &Path) -> bool {
+    path.is_file()
+}
+
+/// Returns true if and only if the given walkdir entry points to a directory.
+///
+/// This works around a bug in Rust's standard library:
+/// https://github.com/rust-lang/rust/issues/46484
+#[cfg(windows)]
+fn walkdir_entry_is_dir(dent: &walkdir::DirEntry) -> bool {
+    dent.metadata().map(|md| metadata_is_dir(&md)).unwrap_or(false)
+}
+
+/// Returns true if and only if the given walkdir entry points to a directory.
+#[cfg(not(windows))]
+fn walkdir_entry_is_dir(dent: &walkdir::DirEntry) -> bool {
+    dent.file_type().is_dir()
+}
+
+/// Returns true if and only if the given metadata points to a directory.
+///
+/// This works around a bug in Rust's standard library:
+/// https://github.com/rust-lang/rust/issues/46484
+#[cfg(windows)]
+fn metadata_is_dir(md: &fs::Metadata) -> bool {
+    use std::os::windows::fs::MetadataExt;
+    use winapi::um::winnt::FILE_ATTRIBUTE_DIRECTORY;
+    md.file_attributes() & FILE_ATTRIBUTE_DIRECTORY != 0
+}
+
 #[cfg(test)]
 mod tests {
     use std::fs::{self, File};

ignore/src/walk.rs

@@ -209,7 +209,7 @@ impl Args {
     /// Returns true if there is exactly one file path given to search.
     pub fn is_one_path(&self) -> bool {
         self.paths.len() == 1
-        && (self.paths[0] == Path::new("-") || self.paths[0].is_file())
+        && (self.paths[0] == Path::new("-") || path_is_file(&self.paths[0]))
     }
 
     /// Create a worker whose configuration is taken from the
@@ -562,7 +562,7 @@ impl<'a> ArgMatches<'a> {
             self.is_present("with-filename")
             || self.is_present("vimgrep")
             || paths.len() > 1
-            || paths.get(0).map_or(false, |p| p.is_dir())
+            || paths.get(0).map_or(false, |p| path_is_dir(p))
         }
     }
 
@@ -609,7 +609,7 @@ impl<'a> ArgMatches<'a> {
         } else {
             // If we're only searching a few paths and all of them are
             // files, then memory maps are probably faster.
-            paths.len() <= 10 && paths.iter().all(|p| p.is_file())
+            paths.len() <= 10 && paths.iter().all(|p| path_is_file(p))
         })
     }
 
@@ -1036,3 +1036,44 @@ fn stdin_is_readable() -> bool {
     // always return true.
     true
 }
+
+/// Returns true if and only if this path points to a directory.
+///
+/// This works around a bug in Rust's standard library:
+/// https://github.com/rust-lang/rust/issues/46484
+#[cfg(windows)]
+fn path_is_dir(path: &Path) -> bool {
+    fs::metadata(path).map(|md| metadata_is_dir(&md)).unwrap_or(false)
+}
+
+/// Returns true if and only if this entry points to a directory.
+#[cfg(not(windows))]
+fn path_is_dir(path: &Path) -> bool {
+    path.is_dir()
+}
+
+/// Returns true if and only if this path points to a file.
+///
+/// This works around a bug in Rust's standard library:
+/// https://github.com/rust-lang/rust/issues/46484
+#[cfg(windows)]
+fn path_is_file(path: &Path) -> bool {
+    !path_is_dir(path)
+}
+
+/// Returns true if and only if this entry points to a directory.
+#[cfg(not(windows))]
+fn path_is_file(path: &Path) -> bool {
+    path.is_file()
+}
+
+/// Returns true if and only if the given metadata points to a directory.
+///
+/// This works around a bug in Rust's standard library:
+/// https://github.com/rust-lang/rust/issues/46484
+#[cfg(windows)]
+fn metadata_is_dir(md: &fs::Metadata) -> bool {
+    use std::os::windows::fs::MetadataExt;
+    use winapi::um::winnt::FILE_ATTRIBUTE_DIRECTORY;
+    md.file_attributes() & FILE_ATTRIBUTE_DIRECTORY != 0
+}

src/args.rs

@@ -18,6 +18,8 @@ extern crate num_cpus;
 extern crate regex;
 extern crate same_file;
 extern crate termcolor;
+#[cfg(windows)]
+extern crate winapi;
 
 use std::error::Error;
 use std::process;
@@ -267,15 +269,14 @@ fn get_or_log_dir_entry(
                     eprintln!("{}", err);
                 }
             }
-            let ft = match dent.file_type() {
-                None => return Some(dent), // entry is stdin
-                Some(ft) => ft,
-            };
+            if dent.file_type().is_none() {
+                return Some(dent); // entry is stdin
+            }
             // A depth of 0 means the user gave the path explicitly, so we
             // should always try to search it.
-            if dent.depth() == 0 && !ft.is_dir() {
+            if dent.depth() == 0 && !ignore_entry_is_dir(&dent) {
                 return Some(dent);
-            } else if !ft.is_file() {
+            } else if !ignore_entry_is_file(&dent) {
                 return None;
             }
             // If we are redirecting stdout to a file, then don't search that
@@ -288,6 +289,45 @@ fn get_or_log_dir_entry(
     }
 }
 
+/// Returns true if and only if the given `ignore::DirEntry` points to a
+/// directory.
+///
+/// This works around a bug in Rust's standard library:
+/// https://github.com/rust-lang/rust/issues/46484
+#[cfg(windows)]
+fn ignore_entry_is_dir(dent: &ignore::DirEntry) -> bool {
+    use std::os::windows::fs::MetadataExt;
+    use winapi::um::winnt::FILE_ATTRIBUTE_DIRECTORY;
+
+    dent.metadata().map(|md| {
+        md.file_attributes() & FILE_ATTRIBUTE_DIRECTORY != 0
+    }).unwrap_or(false)
+}
+
+/// Returns true if and only if the given `ignore::DirEntry` points to a
+/// directory.
+#[cfg(not(windows))]
+fn ignore_entry_is_dir(dent: &ignore::DirEntry) -> bool {
+    dent.file_type().map_or(false, |ft| ft.is_dir())
+}
+
+/// Returns true if and only if the given `ignore::DirEntry` points to a
+/// file.
+///
+/// This works around a bug in Rust's standard library:
+/// https://github.com/rust-lang/rust/issues/46484
+#[cfg(windows)]
+fn ignore_entry_is_file(dent: &ignore::DirEntry) -> bool {
+    !ignore_entry_is_dir(dent)
+}
+
+/// Returns true if and only if the given `ignore::DirEntry` points to a
+/// file.
+#[cfg(not(windows))]
+fn ignore_entry_is_file(dent: &ignore::DirEntry) -> bool {
+    dent.file_type().map_or(false, |ft| ft.is_file())
+}
+
 fn is_stdout_file(
     dent: &ignore::DirEntry,
     stdout_handle: Option<&same_file::Handle>,