(PowerShellGH-879) Use globbing when enumerating workspace files

Previously the Workspace.EnumeratePSFiles method could only filter files
based on file extension (*.ps1, *.psm1, *.psd1).  However editor settings tend
to use file glob patterns, but Editor Services did not have a library that could
parse them.

This commit:
* Updates Editor Services to use the Microsoft.Extensions.FileSystemGlobbing
  library
* Updated the build process to include the new FileSystemGlobbing DLL
* The FileSystemGlobbing library uses an abstract file system to search, not
  an actual System.IO.FileSystem object. So to implement the same error handling
  and maximum depth recursion, a WorkspaceFileSystemWrapperFactory is used to
  create the Directory and File objects needed for the globbing library

  The WorkspaceFileSystemWrapperFactory can filter on:
  - Maximum recursion depth
  - Reparse points (Note that these aren't strictly Symlinks on windows.  There
    are many other types of filesystem items which are reparse points
  - File system extension
  - Gracefully ignores any file access errors

* The EnumeratePSFiles has two method signatures.  One with no arguments which
  uses the Workspace object's default values and another where all arguments
  must be specified when enumerating the files

* Adds tests for the EnumeratePSFiles method to ensure that it filters on glob
  and recursion depth.

Author: glennsarti

PowerShellEditorServices.build.ps1

@@ -57,6 +57,7 @@ $script:RequiredBuildAssets = @{
             'publish/Serilog.Sinks.Async.dll',
             'publish/Serilog.Sinks.Console.dll',
             'publish/Serilog.Sinks.File.dll',
+            'publish/Microsoft.Extensions.FileSystemGlobbing.dll',
             'Microsoft.PowerShell.EditorServices.dll',
             'Microsoft.PowerShell.EditorServices.pdb'
         )

src/PowerShellEditorServices/PowerShellEditorServices.csproj

@@ -20,6 +20,7 @@
     <PackageReference Include="Serilog.Sinks.File" Version="4.0.0" />
     <PackageReference Include="Serilog.Sinks.Async" Version="1.3.0" />
     <PackageReference Include="System.Runtime.Extensions" Version="4.3.0" />
+    <PackageReference Include="Microsoft.Extensions.FileSystemGlobbing" Version="2.2.0" />
     <PackageReference Include="System.Runtime.InteropServices.RuntimeInformation" Version="4.3.0" />
   </ItemGroup>
   <PropertyGroup Condition=" '$(Configuration)' == 'Release' ">

src/PowerShellEditorServices/Workspace/Workspace.cs

@@ -11,6 +11,8 @@
 using System.Security;
 using System.Text;
 using System.Runtime.InteropServices;
+using Microsoft.Extensions.FileSystemGlobbing;
+using Microsoft.Extensions.FileSystemGlobbing.Abstractions;
 
 namespace Microsoft.PowerShell.EditorServices
 {
@@ -22,11 +24,28 @@ public class Workspace
     {
         #region Private Fields
 
-        private static readonly string[] s_psFilePatterns = new []
+        private static readonly string[] s_psFileExtensions =
         {
-            "*.ps1",
-            "*.psm1",
-            "*.psd1"
+            ".ps1",
+            ".psm1",
+            ".psd1"
+        };
+
+        // .Net Core doesn't appear to use the same three letter pattern matching rule although the docs
+        // suggest it should be find the '.ps1xml' files because we search for the pattern '*.ps1'.
+        // ref https://docs.microsoft.com/en-us/dotnet/api/system.io.directory.getfiles?view=netcore-2.1#System_IO_Directory_GetFiles_System_String_System_String_System_IO_EnumerationOptions_
+        private static readonly string[] s_psFileExtensionsDotNetFramework =
+        {
+            ".ps1",
+            ".psm1",
+            ".psd1",
+            ".ps1xml"
+        };
+
+        // An array of globs which includes everything.
+        private static readonly string[] s_psIncludeAllGlob = new []
+        {
+            "**/*"
         };
 
         private ILogger logger;
@@ -42,6 +61,16 @@ public class Workspace
         /// </summary>
         public string WorkspacePath { get; set; }
 
+        /// <summary>
+        /// Gets or sets the default list of file globs to exclude during workspace searches.
+        /// </summary>
+        public List<string> ExcludeFilesGlob { get; set; }
+
+        /// <summary>
+        /// Gets or sets whether the workspace should follow symlinks in search operations.
+        /// </summary>
+        public bool FollowSymlinks { get; set; }
+
         #endregion
 
         #region Constructors
@@ -55,6 +84,8 @@ public Workspace(Version powerShellVersion, ILogger logger)
         {
             this.powerShellVersion = powerShellVersion;
             this.logger = logger;
+            this.ExcludeFilesGlob = new List<string>();
+            this.FollowSymlinks = true;
         }
 
         #endregion
@@ -282,135 +313,64 @@ public string GetRelativePath(string filePath)
         }
 
         /// <summary>
-        /// Enumerate all the PowerShell (ps1, psm1, psd1) files in the workspace in a recursive manner
+        /// Whether the current runtime is Dot Net Framework as opposed to Core or Native
         /// </summary>
-        /// <returns>An enumerator over the PowerShell files found in the workspace</returns>
-        public IEnumerable<string> EnumeratePSFiles()
+        private bool IsDotNetFrameWork()
         {
-            if (WorkspacePath == null || !Directory.Exists(WorkspacePath))
-            {
-                return Enumerable.Empty<string>();
-            }
-
-            var foundFiles = new List<string>();
-            this.RecursivelyEnumerateFiles(WorkspacePath, ref foundFiles);
-            return foundFiles;
+            return RuntimeInformation.FrameworkDescription.StartsWith(".NET Framework", StringComparison.Ordinal);
         }
 
-        #endregion
-
-        #region Private Methods
-
         /// <summary>
-        /// Find PowerShell files recursively down from a given directory path.
-        /// Currently collects files in depth-first order.
-        /// Directory.GetFiles(folderPath, pattern, SearchOption.AllDirectories) would provide this,
-        /// but a cycle in the filesystem will cause that to enter an infinite loop.
+        /// Enumerate all the PowerShell (ps1, psm1, psd1) files in the workspace in a recursive manner, using default values.
         /// </summary>
-        /// <param name="folderPath">The path of the current directory to find files in</param>
-        /// <param name="foundFiles">The accumulator for files found so far.</param>
-        /// <param name="currDepth">The current depth of the recursion from the original base directory.</param>
-        private void RecursivelyEnumerateFiles(string folderPath, ref List<string> foundFiles, int currDepth = 0)
+        /// <returns>An enumerator over the PowerShell files found in the workspace.</returns>
+        public IEnumerable<string> EnumeratePSFiles()
         {
-            const int recursionDepthLimit = 64;
-
-            // Look for any PowerShell files in the current directory
-            foreach (string pattern in s_psFilePatterns)
-            {
-                string[] psFiles;
-                try
-                {
-                    psFiles = Directory.GetFiles(folderPath, pattern, SearchOption.TopDirectoryOnly);
-                }
-                catch (DirectoryNotFoundException e)
-                {
-                    this.logger.WriteHandledException(
-                        $"Could not enumerate files in the path '{folderPath}' due to it being an invalid path",
-                        e);
-
-                    continue;
-                }
-                catch (PathTooLongException e)
-                {
-                    this.logger.WriteHandledException(
-                        $"Could not enumerate files in the path '{folderPath}' due to the path being too long",
-                        e);
-
-                    continue;
-                }
-                catch (Exception e) when (e is SecurityException || e is UnauthorizedAccessException)
-                {
-                    this.logger.WriteHandledException(
-                        $"Could not enumerate files in the path '{folderPath}' due to the path not being accessible",
-                        e);
-
-                    continue;
-                }
-                catch (Exception e)
-                {
-                    this.logger.WriteHandledException(
-                        $"Could not enumerate files in the path '{folderPath}' due to an exception",
-                        e);
-
-                    continue;
-                }
-
-                foundFiles.AddRange(psFiles);
-            }
-
-            // Prevent unbounded recursion here
-            if (currDepth >= recursionDepthLimit)
-            {
-                this.logger.Write(LogLevel.Warning, $"Recursion depth limit hit for path {folderPath}");
-                return;
-            }
-
-            // Add the recursive directories to search next
-            string[] subDirs;
-            try
-            {
-                subDirs = Directory.GetDirectories(folderPath);
-            }
-            catch (DirectoryNotFoundException e)
-            {
-                this.logger.WriteHandledException(
-                    $"Could not enumerate directories in the path '{folderPath}' due to it being an invalid path",
-                    e);
-
-                return;
-            }
-            catch (PathTooLongException e)
-            {
-                this.logger.WriteHandledException(
-                    $"Could not enumerate directories in the path '{folderPath}' due to the path being too long",
-                    e);
-
-                return;
-            }
-            catch (Exception e) when (e is SecurityException || e is UnauthorizedAccessException)
-            {
-                this.logger.WriteHandledException(
-                    $"Could not enumerate directories in the path '{folderPath}' due to the path not being accessible",
-                    e);
+            return EnumeratePSFiles(
+                ExcludeFilesGlob.ToArray(),
+                s_psIncludeAllGlob,
+                maxDepth: 64,
+                ignoreReparsePoints: !FollowSymlinks
+            );
+        }
 
-                return;
-            }
-            catch (Exception e)
+        /// <summary>
+        /// Enumerate all the PowerShell (ps1, psm1, psd1) files in the workspace in a recursive manner.
+        /// </summary>
+        /// <returns>An enumerator over the PowerShell files found in the workspace.</returns>
+        public IEnumerable<string> EnumeratePSFiles(
+            string[] excludeGlobs,
+            string[] includeGlobs,
+            int maxDepth,
+            bool ignoreReparsePoints
+        )
+        {
+            if (WorkspacePath == null || !Directory.Exists(WorkspacePath))
             {
-                this.logger.WriteHandledException(
-                    $"Could not enumerate directories in the path '{folderPath}' due to an exception",
-                    e);
-
-                return;
+                yield break;
             }
 
-
-            foreach (string subDir in subDirs)
+            var matcher = new Microsoft.Extensions.FileSystemGlobbing.Matcher();
+            foreach (string pattern in includeGlobs) { matcher.AddInclude(pattern); }
+            foreach (string pattern in excludeGlobs) { matcher.AddExclude(pattern); }
+
+            var fsFactory = new WorkspaceFileSystemWrapperFactory(
+                WorkspacePath,
+                maxDepth,
+                this.IsDotNetFrameWork() ? s_psFileExtensionsDotNetFramework : s_psFileExtensions,
+                ignoreReparsePoints,
+                this.logger
+            );
+            var result = matcher.Execute(fsFactory.RootDirectory);
+            foreach (FilePatternMatch item in result.Files)
             {
-                RecursivelyEnumerateFiles(subDir, ref foundFiles, currDepth: currDepth + 1);
+                yield return Path.Combine(WorkspacePath, item.Path.Replace('/', Path.DirectorySeparatorChar));
             }
         }
 
+        #endregion
+
+        #region Private Methods
         /// <summary>
         /// Recusrively searches through referencedFiles in scriptFiles
         /// and builds a Dictonary of the file references