Skip to content

Commit f9c404a

Browse files
glennsartiglennsarti
committed
(PowerShellGH-813) Use AST for code folding
The AST contains the most correct version of how a script is interpreted. This includes regions of text. Currently the code folder only uses the Tokens which requires the folder to re-implement some of the AST behaviour e.g. matching token pairs for arrays etc. The code folder should be implemented using as much of the AST as possible. This commit; * Moves most of the region detection to use the AST Extents and uses a new FindFoldsASTVisitor. * Modifies the tests and language server to use the new method fold detection class. * Moved the code to modify the end line of folding regions to the language server code. * The test fixture changes were needed due to how the tokeniser and ast see the beginning of some regions. Users will probably not notice. Note that this requires a modern PowerShell version due to use of ASTVisitor2 class.
1 parent 91f9b1a commit f9c404a

File tree

6 files changed

+219
-60
lines changed

6 files changed

+219
-60
lines changed

‎src/PowerShellEditorServices.Protocol/Server/LanguageServer.cs

+3-1
Original file line numberDiff line numberDiff line change
@@ -1388,7 +1388,9 @@ private FoldingRange[] Fold(string documentUri)
13881388
// If we're showing the last line, decrement the Endline of all regions by one.
13891389
int endLineOffset = this.currentSettings.CodeFolding.ShowLastLine ? -1 : 0;
13901390

1391-
foreach (FoldingReference fold in TokenOperations.FoldableReferences(scriptFile.ScriptTokens).References)
1391+
foreach (FoldingReference fold in FoldingOperations.FoldableRegions(
1392+
scriptFile.ScriptTokens,
1393+
scriptFile.ScriptAst).References)
13921394
{
13931395
result.Add(new FoldingRange {
13941396
EndCharacter = fold.EndCharacter,

‎src/PowerShellEditorServices/Language/AstOperations.cs

+12
Original file line numberDiff line numberDiff line change
@@ -339,5 +339,17 @@ static public string[] FindDotSourcedIncludes(Ast scriptAst, string psScriptRoot
339339

340340
return dotSourcedVisitor.DotSourcedFiles.ToArray();
341341
}
342+
343+
/// <summary>
344+
/// Finds all foldable regions in a script based on AST
345+
/// </summary>
346+
/// <param name="scriptAst">The abstract syntax tree of the given script</param>
347+
/// <param name="refList">The FoldingReferenceList object to add the folds to</param>
348+
/// <returns>A collection of FoldingReference objects</returns>
349+
public static void FindFoldsInDocument(Ast scriptAst, ref FoldingReferenceList refList)
350+
{
351+
FindFoldsVisitor findFoldsVisitor = new FindFoldsVisitor(ref refList);
352+
scriptAst.Visit(findFoldsVisitor);
353+
}
342354
}
343355
}

‎src/PowerShellEditorServices/Language/FindFoldsVisitor.cs

+158
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
//
2+
// Copyright (c) Microsoft. All rights reserved.
3+
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
4+
//
5+
6+
using System;
7+
using System.Collections.Generic;
8+
using System.Management.Automation.Language;
9+
10+
namespace Microsoft.PowerShell.EditorServices
11+
{
12+
/// <summary>
13+
/// The visitor used to find the all folding regions in an AST
14+
/// </summary>
15+
internal class FindFoldsVisitor : AstVisitor2
16+
{
17+
private const string RegionKindNone = null;
18+
19+
private FoldingReferenceList _refList;
20+
21+
public FindFoldsVisitor(ref FoldingReferenceList refList)
22+
{
23+
_refList = refList;
24+
}
25+
26+
/// <summary>
27+
/// Returns whether an Extent could be used as a valid folding region
28+
/// </summary>
29+
private bool IsValidFoldingExtent(
30+
IScriptExtent extent)
31+
{
32+
// The extent must span at least one line
33+
return extent.EndLineNumber > extent.StartLineNumber;
34+
}
35+
36+
/// <summary>
37+
/// Creates an instance of a FoldingReference object from a script extent
38+
/// </summary>
39+
private FoldingReference CreateFoldingReference(
40+
IScriptExtent extent,
41+
string matchKind)
42+
{
43+
// Extents are base 1, but LSP is base 0, so minus 1 off all lines and character positions
44+
return new FoldingReference
45+
{
46+
StartLine = extent.StartLineNumber - 1,
47+
StartCharacter = extent.StartColumnNumber - 1,
48+
EndLine = extent.EndLineNumber - 1,
49+
EndCharacter = extent.EndColumnNumber - 1,
50+
Kind = matchKind
51+
};
52+
}
53+
54+
// AST object visitor methods
55+
public override AstVisitAction VisitArrayExpression(ArrayExpressionAst objAst)
56+
{
57+
if (IsValidFoldingExtent(objAst.Extent))
58+
{
59+
_refList.SafeAdd(CreateFoldingReference(objAst.Extent, RegionKindNone));
60+
}
61+
return AstVisitAction.Continue;
62+
}
63+
64+
public override AstVisitAction VisitHashtable(HashtableAst objAst)
65+
{
66+
if (IsValidFoldingExtent(objAst.Extent))
67+
{
68+
_refList.SafeAdd(CreateFoldingReference(objAst.Extent, RegionKindNone));
69+
}
70+
return AstVisitAction.Continue;
71+
}
72+
73+
public override AstVisitAction VisitParamBlock(ParamBlockAst objAst)
74+
{
75+
if (IsValidFoldingExtent(objAst.Extent)) { _refList.SafeAdd(CreateFoldingReference(objAst.Extent, RegionKindNone)); }
76+
return AstVisitAction.Continue;
77+
}
78+
79+
public override AstVisitAction VisitStatementBlock(StatementBlockAst objAst)
80+
{
81+
// These parent visitors will get this AST Object. No need to process it
82+
if (objAst.Parent == null) { return AstVisitAction.Continue; }
83+
if (objAst.Parent is ArrayExpressionAst) { return AstVisitAction.Continue; }
84+
if (IsValidFoldingExtent(objAst.Extent))
85+
{
86+
_refList.SafeAdd(CreateFoldingReference(objAst.Extent, RegionKindNone));
87+
}
88+
return AstVisitAction.Continue;
89+
}
90+
91+
public override AstVisitAction VisitScriptBlock(ScriptBlockAst objAst)
92+
{
93+
// If the Parent object is null then this represents the entire script. We don't want to fold that
94+
if (objAst.Parent == null) { return AstVisitAction.Continue; }
95+
// The ScriptBlockExpressionAst visitor will get this AST Object. No need to process it
96+
if (objAst.Parent is ScriptBlockExpressionAst) { return AstVisitAction.Continue; }
97+
if (IsValidFoldingExtent(objAst.Extent))
98+
{
99+
_refList.SafeAdd(CreateFoldingReference(objAst.Extent, RegionKindNone));
100+
}
101+
return AstVisitAction.Continue;
102+
}
103+
104+
public override AstVisitAction VisitScriptBlockExpression(ScriptBlockExpressionAst objAst)
105+
{
106+
if (IsValidFoldingExtent(objAst.Extent)) {
107+
FoldingReference foldRef = CreateFoldingReference(objAst.ScriptBlock.Extent, RegionKindNone);
108+
if (objAst.Parent == null) { return AstVisitAction.Continue; }
109+
if (objAst.Parent is InvokeMemberExpressionAst) {
110+
// This is a bit naive. The ScriptBlockExpressionAst Extent does not include the actual { and }
111+
// characters so the StartCharacter and EndCharacter indexes are out by one. This could be a bug in
112+
// PowerShell Parser. This is just a workaround
113+
foldRef.StartCharacter--;
114+
foldRef.EndCharacter++;
115+
}
116+
_refList.SafeAdd(foldRef);
117+
}
118+
return AstVisitAction.Continue;
119+
}
120+
121+
public override AstVisitAction VisitStringConstantExpression(StringConstantExpressionAst objAst)
122+
{
123+
if (IsValidFoldingExtent(objAst.Extent))
124+
{
125+
_refList.SafeAdd(CreateFoldingReference(objAst.Extent, RegionKindNone));
126+
}
127+
128+
return AstVisitAction.Continue;
129+
}
130+
131+
public override AstVisitAction VisitSubExpression(SubExpressionAst objAst)
132+
{
133+
if (IsValidFoldingExtent(objAst.Extent))
134+
{
135+
_refList.SafeAdd(CreateFoldingReference(objAst.Extent, RegionKindNone));
136+
}
137+
return AstVisitAction.Continue;
138+
}
139+
140+
public override AstVisitAction VisitTypeDefinition(TypeDefinitionAst objAst)
141+
{
142+
if (IsValidFoldingExtent(objAst.Extent))
143+
{
144+
_refList.SafeAdd(CreateFoldingReference(objAst.Extent, RegionKindNone));
145+
}
146+
return AstVisitAction.Continue;
147+
}
148+
149+
public override AstVisitAction VisitVariableExpression(VariableExpressionAst objAst)
150+
{
151+
if (IsValidFoldingExtent(objAst.Extent))
152+
{
153+
_refList.SafeAdd(CreateFoldingReference(objAst.Extent, RegionKindNone));
154+
}
155+
return AstVisitAction.Continue;
156+
}
157+
}
158+
}

‎src/PowerShellEditorServices/Language/FoldingOperations.cs

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
//
2+
// Copyright (c) Microsoft. All rights reserved.
3+
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
4+
//
5+
6+
using System;
7+
using System.Collections.Generic;
8+
using System.Management.Automation.Language;
9+
10+
namespace Microsoft.PowerShell.EditorServices
11+
{
12+
/// <summary>
13+
/// Provides common operations for code folding in a script
14+
/// </summary>
15+
internal static class FoldingOperations
16+
{
17+
/// <summary>
18+
/// Extracts all of the unique foldable regions in a script given a script AST and the list tokens
19+
/// used to generate the AST
20+
/// </summary>
21+
internal static FoldingReferenceList FoldableRegions(
22+
Token[] tokens,
23+
Ast scriptAst)
24+
{
25+
var foldableRegions = new FoldingReferenceList();
26+
27+
// Add regions from AST
28+
AstOperations.FindFoldsInDocument(scriptAst, ref foldableRegions);
29+
30+
// Add regions from Tokens
31+
TokenOperations.FoldableReferences(tokens, ref foldableRegions);
32+
33+
return foldableRegions;
34+
}
35+
}
36+
}

‎src/PowerShellEditorServices/Language/TokenOperations.cs

+3-56
Original file line numberDiff line numberDiff line change
@@ -33,61 +33,10 @@ internal static class TokenOperations
3333
/// <summary>
3434
/// Extracts all of the unique foldable regions in a script given the list tokens
3535
/// </summary>
36-
internal static FoldingReferenceList FoldableReferences(
37-
Token[] tokens)
36+
internal static void FoldableReferences(
37+
Token[] tokens,
38+
ref FoldingReferenceList refList)
3839
{
39-
var refList = new FoldingReferenceList();
40-
41-
Stack<Token> tokenCurlyStack = new Stack<Token>();
42-
Stack<Token> tokenParenStack = new Stack<Token>();
43-
foreach (Token token in tokens)
44-
{
45-
switch (token.Kind)
46-
{
47-
// Find matching braces { -> }
48-
// Find matching hashes @{ -> }
49-
case TokenKind.LCurly:
50-
case TokenKind.AtCurly:
51-
tokenCurlyStack.Push(token);
52-
break;
53-
54-
case TokenKind.RCurly:
55-
if (tokenCurlyStack.Count > 0)
56-
{
57-
refList.SafeAdd(CreateFoldingReference(tokenCurlyStack.Pop(), token, RegionKindNone));
58-
}
59-
break;
60-
61-
// Find matching parentheses ( -> )
62-
// Find matching array literals @( -> )
63-
// Find matching subexpressions $( -> )
64-
case TokenKind.LParen:
65-
case TokenKind.AtParen:
66-
case TokenKind.DollarParen:
67-
tokenParenStack.Push(token);
68-
break;
69-
70-
case TokenKind.RParen:
71-
if (tokenParenStack.Count > 0)
72-
{
73-
refList.SafeAdd(CreateFoldingReference(tokenParenStack.Pop(), token, RegionKindNone));
74-
}
75-
break;
76-
77-
// Find contiguous here strings @' -> '@
78-
// Find unopinionated variable names ${ \n \n }
79-
// Find contiguous expandable here strings @" -> "@
80-
case TokenKind.HereStringLiteral:
81-
case TokenKind.Variable:
82-
case TokenKind.HereStringExpandable:
83-
if (token.Extent.StartLineNumber != token.Extent.EndLineNumber)
84-
{
85-
refList.SafeAdd(CreateFoldingReference(token, token, RegionKindNone));
86-
}
87-
break;
88-
}
89-
}
90-
9140
// Find matching comment regions #region -> #endregion
9241
// Given a list of tokens, find the tokens that are comments and
9342
// the comment text is either `#region` or `#endregion`, and then use a stack to determine
@@ -153,8 +102,6 @@ internal static FoldingReferenceList FoldableReferences(
153102
{
154103
refList.SafeAdd(CreateFoldingReference(blockStartToken, blockNextLine - 1, RegionKindComment));
155104
}
156-
157-
return refList;
158105
}
159106

160107
/// <summary>

‎test/PowerShellEditorServices.Test/Language/TokenOperationsTests.cs

+7-3
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,9 @@ private FoldingReference[] GetRegions(string text) {
2020
text,
2121
Version.Parse("5.0"));
2222

23-
var result = Microsoft.PowerShell.EditorServices.TokenOperations.FoldableReferences(scriptFile.ScriptTokens).ToArray();
23+
var result = Microsoft.PowerShell.EditorServices.FoldingOperations.FoldableRegions(
24+
scriptFile.ScriptTokens,
25+
scriptFile.ScriptAst).ToArray();
2426
// The foldable regions need to be deterministic for testing so sort the array.
2527
Array.Sort(result);
2628
return result;
@@ -256,6 +258,7 @@ public void LaguageServiceFindsFoldablRegionsWithSameEndToken() {
256258
}
257259

258260
// A simple PowerShell Classes test
261+
[Trait("Category", "Folding")]
259262
[Fact]
260263
public void LaguageServiceFindsFoldablRegionsWithClasses() {
261264
string testString =
@@ -271,7 +274,7 @@ [string] TestMethod() {
271274
}
272275
";
273276
FoldingReference[] expectedFolds = {
274-
CreateFoldingReference(0, 16, 9, 1, null),
277+
CreateFoldingReference(0, 0, 9, 1, null),
275278
CreateFoldingReference(1, 31, 4, 16, null),
276279
CreateFoldingReference(6, 26, 8, 5, null)
277280
};
@@ -282,6 +285,7 @@ [string] TestMethod() {
282285
}
283286

284287
// This tests DSC style keywords and param blocks
288+
[Trait("Category", "Folding")]
285289
[Fact]
286290
public void LaguageServiceFindsFoldablRegionsWithDSC() {
287291
string testString =
@@ -322,7 +326,7 @@ AdcsCertificationAuthority CertificateAuthority
322326
";
323327
FoldingReference[] expectedFolds = {
324328
CreateFoldingReference(1, 0, 33, 1, null),
325-
CreateFoldingReference(3, 4, 12, 5, null),
329+
CreateFoldingReference(2, 4, 12, 5, null),
326330
CreateFoldingReference(17, 4, 32, 5, null),
327331
CreateFoldingReference(19, 8, 22, 9, null),
328332
CreateFoldingReference(25, 8, 31, 9, null)

0 commit comments

Comments
 (0)