Skip to content

Add phrase_limit to highlighting #2959

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Dec 19, 2017
Merged

Add phrase_limit to highlighting #2959

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
8adfa8a
Add phrase_limit to highlighting
russcam Dec 15, 2017
4809caf
Change RequireFieldMatch to bool? with default value
russcam Dec 18, 2017
95782d3
Merge branch 'master' into fix/highlighting-phrase-limit
russcam Dec 18, 2017
8ba4974
Merge branch 'master' into fix/highlighting-phrase-limit
russcam Dec 19, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions docs/search/request/highlighting-usage.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ s => s
.Highlight(h => h
.PreTags("<tag1>")
.PostTags("</tag1>")
.Encoder(HighlighterEncoder.Html)
.Fields(
fs => fs
.Field(p => p.Name.Suffix("standard"))
Expand All @@ -50,6 +51,7 @@ s => s
.PreTags("<name>")
.PostTags("</name>")
.BoundaryMaxScan(50)
.PhraseLimit(10)
.HighlightQuery(q => q
.Match(m => m
.Field(p => p.LeadDeveloper.FirstName)
Expand Down Expand Up @@ -87,6 +89,7 @@ new SearchRequest<Project>
{
PreTags = new[] { "<tag1>" },
PostTags = new[] { "</tag1>" },
Encoder = HighlighterEncoder.Html,
Fields = new Dictionary<Field, IHighlightField>
{
{ "name.standard", new HighlightField
Expand All @@ -102,6 +105,7 @@ new SearchRequest<Project>
{ "leadDeveloper.firstName", new HighlightField
{
Type = "fvh",
PhraseLimit = 10,
BoundaryMaxScan = 50,
PreTags = new[] { "<name>"},
PostTags = new[] { "</name>"},
Expand Down Expand Up @@ -147,6 +151,7 @@ new SearchRequest<Project>
"post_tags": [
"</tag1>"
],
"encoder": "html",
"fields": {
"name.standard": {
"type": "plain",
Expand All @@ -158,6 +163,7 @@ new SearchRequest<Project>
},
"leadDeveloper.firstName": {
"type": "fvh",
"phrase_limit": 10,
"boundary_max_scan": 50,
"pre_tags": [
"<name>"
Expand Down
47 changes: 18 additions & 29 deletions src/Nest/Search/Search/Highlighting/Highlight.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,10 +54,6 @@ public interface IHighlight
[JsonProperty("fragment_offset")]
int? FragmentOffset { get; set; }

[Obsolete("Bad mapping use BoundaryMaxScan instead")]
[JsonProperty("boundary_max_size")]
int? BoundaryMaxSize { get; set; }

/// <summary>
/// Controls how far to look for boundary characters. Defaults to 20.
/// </summary>
Expand All @@ -69,13 +65,13 @@ public interface IHighlight
/// It can be either default (no encoding) or html (will escape html, if you use html highlighting tags).
/// </summary>
[JsonProperty("encoder")]
string Encoder { get; set; }
HighlighterEncoder? Encoder { get; set; }

/// <summary>
/// The order in which highlighted fragments are sorted
/// </summary>
[JsonProperty("order")]
string Order { get; set; }
HighlighterOrder? Order { get; set; }

/// <summary>
/// Use a specific "tag" schemas.
Expand All @@ -88,7 +84,7 @@ public interface IHighlight
/// &lt;em class="hlt10"&gt;
/// </remarks>
[JsonProperty("tags_schema")]
string TagsSchema { get; set; }
HighlighterTagsSchema? TagsSchema { get; set; }

[JsonProperty("fields")]
[JsonConverter(typeof(VerbatimDictionaryKeysJsonConverter<Field, IHighlightField>))]
Expand Down Expand Up @@ -147,21 +143,17 @@ public class Highlight : IHighlight
// <inheritdoc/>
public int? FragmentSize { get; set; }
// <inheritdoc/>
public string TagsSchema { get; set; }
public HighlighterTagsSchema? TagsSchema { get; set; }
// <inheritdoc/>
public int? NumberOfFragments { get; set; }
// <inheritdoc/>
public int? FragmentOffset { get; set; }
// <inheritdoc/>
[Obsolete("Bad mapping use BoundaryMaxScan instead")]
// <inheritdoc/>
public int? BoundaryMaxSize { get; set; }
// <inheritdoc/>
public int? BoundaryMaxScan { get; set; }
// <inheritdoc/>
public string Encoder { get; set; }
public HighlighterEncoder? Encoder { get; set; }
// <inheritdoc/>
public string Order { get; set; }
public HighlighterOrder? Order { get; set; }
// <inheritdoc/>
public Dictionary<Field, IHighlightField> Fields { get; set; }
// <inheritdoc/>
Expand All @@ -187,13 +179,12 @@ public class HighlightDescriptor<T> : DescriptorBase<HighlightDescriptor<T> ,IHi
IEnumerable<string> IHighlight.PreTags { get; set; }
IEnumerable<string> IHighlight.PostTags { get; set; }
int? IHighlight.FragmentSize { get; set; }
string IHighlight.TagsSchema { get; set; }
HighlighterTagsSchema? IHighlight.TagsSchema { get; set; }
int? IHighlight.NumberOfFragments { get; set; }
int? IHighlight.FragmentOffset { get; set; }
int? IHighlight.BoundaryMaxSize { get; set; }
int? IHighlight.BoundaryMaxScan { get; set; }
string IHighlight.Encoder { get; set; }
string IHighlight.Order { get; set; }
HighlighterEncoder? IHighlight.Encoder { get; set; }
HighlighterOrder? IHighlight.Order { get; set; }
Dictionary<Field, IHighlightField> IHighlight.Fields { get; set; }
bool? IHighlight.RequireFieldMatch { get; set; }
string IHighlight.BoundaryChars { get; set; }
Expand All @@ -215,13 +206,13 @@ public HighlightDescriptor<T> Fields(params Func<HighlightFieldDescriptor<T>, IH
);

// <inheritdoc/>
public HighlightDescriptor<T> TagsSchema(string schema = "styled") => Assign(a => a.TagsSchema = schema);
public HighlightDescriptor<T> TagsSchema(HighlighterTagsSchema? schema) => Assign(a => a.TagsSchema = schema);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No default value?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I can make it the default, but the client generally doesn't have default values for nullable enums. I suspect this had a default value more as a hint for the accepted string value.


// <inheritdoc/>
public HighlightDescriptor<T> PreTags(string preTags) => this.PreTags(new[] {preTags});

// <inheritdoc/>
public HighlightDescriptor<T> PostTags(string postTags)=> this.PostTags(new[] {postTags});
public HighlightDescriptor<T> PostTags(string postTags) => this.PostTags(new[] {postTags});

// <inheritdoc/>
public HighlightDescriptor<T> PreTags(IEnumerable<string> preTags) => Assign(a => a.PreTags = preTags.ToListOrNullIfEmpty());
Expand All @@ -230,30 +221,28 @@ public HighlightDescriptor<T> Fields(params Func<HighlightFieldDescriptor<T>, IH
public HighlightDescriptor<T> PostTags(IEnumerable<string> postTags) => Assign(a => a.PostTags = postTags.ToListOrNullIfEmpty());

// <inheritdoc/>
public HighlightDescriptor<T> FragmentSize(int fragmentSize) => Assign(a => a.FragmentSize = fragmentSize);
public HighlightDescriptor<T> FragmentSize(int? fragmentSize) => Assign(a => a.FragmentSize = fragmentSize);

// <inheritdoc/>
public HighlightDescriptor<T> NumberOfFragments(int numberOfFragments) => Assign(a => a.NumberOfFragments = numberOfFragments);
public HighlightDescriptor<T> NumberOfFragments(int? numberOfFragments) => Assign(a => a.NumberOfFragments = numberOfFragments);

// <inheritdoc/>
public HighlightDescriptor<T> FragmentOffset(int fragmentOffset) => Assign(a => a.FragmentOffset = fragmentOffset);
public HighlightDescriptor<T> FragmentOffset(int? fragmentOffset) => Assign(a => a.FragmentOffset = fragmentOffset);

// <inheritdoc/>
public HighlightDescriptor<T> Encoder(string encoder) => Assign(a => a.Encoder = encoder);
public HighlightDescriptor<T> Encoder(HighlighterEncoder? encoder) => Assign(a => a.Encoder = encoder);

// <inheritdoc/>
public HighlightDescriptor<T> Order(string order) => Assign(a => a.Order = order);
public HighlightDescriptor<T> Order(HighlighterOrder? order) => Assign(a => a.Order = order);

// <inheritdoc/>
public HighlightDescriptor<T> RequireFieldMatch(bool requireFieldMatch) => Assign(a => a.RequireFieldMatch = requireFieldMatch);
public HighlightDescriptor<T> RequireFieldMatch(bool? requireFieldMatch = true) => Assign(a => a.RequireFieldMatch = requireFieldMatch);

// <inheritdoc/>
public HighlightDescriptor<T> BoundaryCharacters(string boundaryCharacters) => Assign(a => a.BoundaryChars = boundaryCharacters);

[Obsolete("Bad mapping use BoundaryMaxScan instead")]
public HighlightDescriptor<T> BoundaryMaxSize(int boundaryMaxSize) => Assign(a => a.BoundaryMaxSize = boundaryMaxSize);
// <inheritdoc/>
public HighlightDescriptor<T> BoundaryMaxScan(int boundaryMaxScan) => Assign(a => a.BoundaryMaxScan = boundaryMaxScan);
public HighlightDescriptor<T> BoundaryMaxScan(int? boundaryMaxScan) => Assign(a => a.BoundaryMaxScan = boundaryMaxScan);

// <inheritdoc/>
public HighlightDescriptor<T> MaxFragmentLength(int? maxFragmentLength) => Assign(a => a.MaxFragmentLength = maxFragmentLength);
Expand Down
47 changes: 25 additions & 22 deletions src/Nest/Search/Search/Highlighting/HighlightField.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,17 +68,10 @@ public interface IHighlightField
int? BoundaryMaxScan { get; set; }

/// <summary>
/// Define how highlighted text will be encoded.
/// It can be either default (no encoding) or html (will escape html, if you use html highlighting tags).
/// </summary>
[JsonProperty("encoder")]
string Encoder { get; set; }

/// <summary>
/// The order in which highlighted fragments are sorted
/// The order in which highlighted fragments are sorted. Only valid for the unified highlighter.
/// </summary>
[JsonProperty("order")]
string Order { get; set; }
HighlighterOrder? Order { get; set; }

/// <summary>
/// Use a specific "tag" schemas.
Expand All @@ -91,7 +84,7 @@ public interface IHighlightField
/// &lt;em class="hlt10"&gt;
/// </remarks>
[JsonProperty("tags_schema")]
string TagsSchema { get; set; }
HighlighterTagsSchema? TagsSchema { get; set; }

/// <summary>
/// Determines if only fields that hold a query match will be highlighted. Set to <c>false</c>
Expand Down Expand Up @@ -129,6 +122,7 @@ public interface IHighlightField
/// </summary>
[JsonProperty("fragmenter")]
HighlighterFragmenter? Fragmenter { get; set; }

/// <summary>
/// The type of highlighter to use. Can be a defined or custom highlighter
/// </summary>
Expand All @@ -155,6 +149,15 @@ public interface IHighlightField
/// </summary>
[JsonProperty("highlight_query")]
QueryContainer HighlightQuery { get; set; }

/// <summary>
/// Controls the number of matching phrases in a document that are considered. Prevents the
/// <see cref="HighlighterType.Fvh"/> highlighter from analyzing too many phrases and consuming too much memory.
/// When using matched_fields, <see cref="PhraseLimit"/> phrases per matched field are considered. Raising the limit increases query time
/// and consumes more memory. Only supported by the <see cref="HighlighterType.Fvh"/> highlighter. Defaults to 256.
/// </summary>
[JsonProperty("phrase_limit")]
int? PhraseLimit { get; set; }
}

public class HighlightField : IHighlightField
Expand All @@ -176,11 +179,9 @@ public class HighlightField : IHighlightField
/// <inheritdoc/>
public int? BoundaryMaxScan { get; set; }
/// <inheritdoc/>
public string Encoder { get; set; }
public HighlighterOrder? Order { get; set; }
/// <inheritdoc/>
public string Order { get; set; }
/// <inheritdoc/>
public string TagsSchema { get; set; }
public HighlighterTagsSchema? TagsSchema { get; set; }
/// <inheritdoc/>
public bool? RequireFieldMatch { get; set; }
/// <inheritdoc/>
Expand All @@ -201,6 +202,8 @@ public class HighlightField : IHighlightField
public Fields MatchedFields { get; set; }
/// <inheritdoc/>
public QueryContainer HighlightQuery { get; set; }
/// <inheritdoc/>
public int? PhraseLimit { get; set; }
}

public class HighlightFieldDescriptor<T> : DescriptorBase<HighlightFieldDescriptor<T>,IHighlightField>, IHighlightField
Expand All @@ -214,9 +217,8 @@ public class HighlightFieldDescriptor<T> : DescriptorBase<HighlightFieldDescript
int? IHighlightField.NumberOfFragments { get; set; }
int? IHighlightField.FragmentOffset { get; set; }
int? IHighlightField.BoundaryMaxScan { get; set; }
string IHighlightField.Encoder { get; set; }
string IHighlightField.Order { get; set; }
string IHighlightField.TagsSchema { get; set; }
HighlighterOrder? IHighlightField.Order { get; set; }
HighlighterTagsSchema? IHighlightField.TagsSchema { get; set; }
bool? IHighlightField.RequireFieldMatch { get; set; }
string IHighlightField.BoundaryChars { get; set; }
int? IHighlightField.MaxFragmentLength { get; set; }
Expand All @@ -227,6 +229,7 @@ public class HighlightFieldDescriptor<T> : DescriptorBase<HighlightFieldDescript
bool? IHighlightField.ForceSource { get; set; }
Fields IHighlightField.MatchedFields { get; set; }
QueryContainer IHighlightField.HighlightQuery { get; set; }
int? IHighlightField.PhraseLimit { get; set; }

/// <inheritdoc/>
public HighlightFieldDescriptor<T> Field(Field field) => Assign(a => a.Field = field);
Expand All @@ -238,7 +241,7 @@ public class HighlightFieldDescriptor<T> : DescriptorBase<HighlightFieldDescript
public HighlightFieldDescriptor<T> AllField() => this.Field("_all");

/// <inheritdoc/>
public HighlightFieldDescriptor<T> TagsSchema(string schema = "styled") => Assign(a => a.TagsSchema = schema);
public HighlightFieldDescriptor<T> TagsSchema(HighlighterTagsSchema? schema) => Assign(a => a.TagsSchema = schema);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No default?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I can make it the default, but the client generally doesn't have default values for nullable enums. I suspect this had a default value more as a hint for the accepted string value.


/// <inheritdoc/>
public HighlightFieldDescriptor<T> ForceSource(bool? force = true) => Assign(a => a.ForceSource = force);
Expand Down Expand Up @@ -274,10 +277,7 @@ public class HighlightFieldDescriptor<T> : DescriptorBase<HighlightFieldDescript
public HighlightFieldDescriptor<T> FragmentOffset(int? fragmentOffset) => Assign(a => a.FragmentOffset = fragmentOffset);

/// <inheritdoc/>
public HighlightFieldDescriptor<T> Encoder(string encoder) => Assign(a => a.Encoder = encoder);

/// <inheritdoc/>
public HighlightFieldDescriptor<T> Order(string order) => Assign(a => a.Order = order);
public HighlightFieldDescriptor<T> Order(HighlighterOrder? order) => Assign(a => a.Order = order);

/// <inheritdoc/>
public HighlightFieldDescriptor<T> RequireFieldMatch(bool? requireFieldMatch = true) => Assign(a => a.RequireFieldMatch = requireFieldMatch);
Expand Down Expand Up @@ -307,5 +307,8 @@ public HighlightFieldDescriptor<T> HighlightQuery(Func<QueryContainerDescriptor<

/// <inheritdoc/>
public HighlightFieldDescriptor<T> Fragmenter(HighlighterFragmenter? fragmenter) => Assign(a => a.Fragmenter = fragmenter);

/// <inheritdoc/>
public HighlightFieldDescriptor<T> PhraseLimit(int phraseLimit) => Assign(a => a.PhraseLimit = phraseLimit);
}
}
24 changes: 24 additions & 0 deletions src/Nest/Search/Search/Highlighting/HighlighterEncoder.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
using System.Runtime.Serialization;
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;

namespace Nest
{
/// <summary>
/// Indicates if the highlighted text should be HTML encoded
/// </summary>
[JsonConverter(typeof(StringEnumConverter))]
public enum HighlighterEncoder
{
/// <summary>
/// No encoding
/// </summary>
[EnumMember(Value = "default")]
Default,
/// <summary>
/// Escapes HTML highlighting tags
/// </summary>
[EnumMember(Value = "html")]
Html
}
}
19 changes: 19 additions & 0 deletions src/Nest/Search/Search/Highlighting/HighlighterOrder.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
using System.Runtime.Serialization;
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;

namespace Nest
{
/// <summary>
/// Sorts highlighted fragments
/// </summary>
[JsonConverter(typeof(StringEnumConverter))]
public enum HighlighterOrder
{
/// <summary>
/// Sorts highlighted fragments by score. Only valid for the <see cref="HighlighterType.Unified"/> highligher
/// </remarks>
[EnumMember(Value = "score")]
Score
}
}
26 changes: 26 additions & 0 deletions src/Nest/Search/Search/Highlighting/HighlighterTagsSchema.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
using System.Runtime.Serialization;
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;

namespace Nest
{
/// <summary>
/// Use a built-in tag schema
/// </summary>
[JsonConverter(typeof(StringEnumConverter))]
public enum HighlighterTagsSchema
{
/// <summary>
/// Use a specific "tag" schemas.
/// </summary>
/// <remarks>
/// <para>Currently a single schema called "styled" with the following pre_tags:</para>
/// <para>&lt;em class="hlt1"&gt;, &lt;em class="hlt2"&gt;, &lt;em class="hlt3"&gt;,</para>
/// <para>&lt;em class="hlt4"&gt;, &lt;em class="hlt5"&gt;, &lt;em class="hlt6"&gt;,</para>
/// <para>&lt;em class="hlt7"&gt;, &lt;em class="hlt8"&gt;, &lt;em class="hlt9"&gt;,</para>
/// <para>&lt;em class="hlt10"&gt;</para>
/// </remarks>
[EnumMember(Value = "styled")]
Styled
}
}
Loading