Added search using qualifier[:=]value syntax (#2373)

Author: Zach Vonler

Diff for: commands/lib/search.go

@@ -23,7 +23,6 @@ import (
 	"github.com/arduino/arduino-cli/arduino"
 	"github.com/arduino/arduino-cli/arduino/libraries/librariesindex"
 	"github.com/arduino/arduino-cli/arduino/libraries/librariesmanager"
-	"github.com/arduino/arduino-cli/arduino/utils"
 	"github.com/arduino/arduino-cli/commands/internal/instances"
 	rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
 	semver "go.bug.st/relaxed-semver"
@@ -44,18 +43,11 @@ func searchLibrary(req *rpc.LibrarySearchRequest, lm *librariesmanager.Libraries
 	if query == "" {
 		query = req.GetQuery()
 	}
-	queryTerms := utils.SearchTermsFromQueryString(query)
 
-	for _, lib := range lm.Index.Libraries {
-		toTest := lib.Name + " " +
-			lib.Latest.Paragraph + " " +
-			lib.Latest.Sentence + " " +
-			lib.Latest.Author + " "
-		for _, include := range lib.Latest.ProvidesIncludes {
-			toTest += include + " "
-		}
+	matcher := MatcherFromQueryString(query)
 
-		if utils.Match(toTest, queryTerms) {
+	for _, lib := range lm.Index.Libraries {
+		if matcher(lib) {
 			res = append(res, indexLibraryToRPCSearchLibrary(lib, req.GetOmitReleasesDetails()))
 		}
 	}

Diff for: commands/lib/search_matcher.go

@@ -0,0 +1,135 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2023 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to [email protected].
+
+package lib
+
+import (
+	"strings"
+
+	"github.com/arduino/arduino-cli/arduino/libraries/librariesindex"
+	"github.com/arduino/arduino-cli/arduino/utils"
+)
+
+// matcherTokensFromQueryString parses the query string into tokens of interest
+// for the qualifier-value pattern matching.
+func matcherTokensFromQueryString(query string) []string {
+	escaped := false
+	quoted := false
+	tokens := []string{}
+	sb := &strings.Builder{}
+
+	for _, r := range query {
+		// Short circuit the loop on backslash so that all other paths can clear
+		// the escaped flag.
+		if !escaped && r == '\\' {
+			escaped = true
+			continue
+		}
+
+		if r == '"' {
+			if !escaped {
+				quoted = !quoted
+			} else {
+				sb.WriteRune(r)
+			}
+		} else if !quoted && r == ' ' {
+			tokens = append(tokens, strings.ToLower(sb.String()))
+			sb.Reset()
+		} else {
+			sb.WriteRune(r)
+		}
+		escaped = false
+	}
+	if sb.Len() > 0 {
+		tokens = append(tokens, strings.ToLower(sb.String()))
+	}
+
+	return tokens
+}
+
+// defaulLibraryMatchExtractor returns a string describing the library that
+// is used for the simple search.
+func defaultLibraryMatchExtractor(lib *librariesindex.Library) string {
+	res := lib.Name + " " +
+		lib.Latest.Paragraph + " " +
+		lib.Latest.Sentence + " " +
+		lib.Latest.Author + " "
+	for _, include := range lib.Latest.ProvidesIncludes {
+		res += include + " "
+	}
+	return res
+}
+
+var qualifiers map[string]func(lib *librariesindex.Library) string = map[string]func(lib *librariesindex.Library) string{
+	"name":          func(lib *librariesindex.Library) string { return lib.Name },
+	"architectures": func(lib *librariesindex.Library) string { return strings.Join(lib.Latest.Architectures, " ") },
+	"author":        func(lib *librariesindex.Library) string { return lib.Latest.Author },
+	"category":      func(lib *librariesindex.Library) string { return lib.Latest.Category },
+	"dependencies": func(lib *librariesindex.Library) string {
+		names := make([]string, len(lib.Latest.Dependencies))
+		for i, dep := range lib.Latest.Dependencies {
+			names[i] = dep.GetName()
+		}
+		return strings.Join(names, " ")
+	},
+	"license":    func(lib *librariesindex.Library) string { return lib.Latest.License },
+	"maintainer": func(lib *librariesindex.Library) string { return lib.Latest.Maintainer },
+	"paragraph":  func(lib *librariesindex.Library) string { return lib.Latest.Paragraph },
+	"provides":   func(lib *librariesindex.Library) string { return strings.Join(lib.Latest.ProvidesIncludes, " ") },
+	"sentence":   func(lib *librariesindex.Library) string { return lib.Latest.Sentence },
+	"types":      func(lib *librariesindex.Library) string { return strings.Join(lib.Latest.Types, " ") },
+	"version":    func(lib *librariesindex.Library) string { return lib.Latest.Version.String() },
+	"website":    func(lib *librariesindex.Library) string { return lib.Latest.Website },
+}
+
+// MatcherFromQueryString returns a closure that takes a library as a
+// parameter and returns true if the library matches the query.
+func MatcherFromQueryString(query string) func(*librariesindex.Library) bool {
+	// A qv-query is one using <qualifier>[:=]<value> syntax.
+	qvQuery := strings.Contains(query, ":") || strings.Contains(query, "=")
+
+	if !qvQuery {
+		queryTerms := utils.SearchTermsFromQueryString(query)
+		return func(lib *librariesindex.Library) bool {
+			return utils.Match(defaultLibraryMatchExtractor(lib), queryTerms)
+		}
+	}
+
+	queryTerms := matcherTokensFromQueryString(query)
+
+	return func(lib *librariesindex.Library) bool {
+		matched := true
+		for _, term := range queryTerms {
+			if sepIdx := strings.IndexAny(term, ":="); sepIdx != -1 {
+				qualifier, separator, target := term[:sepIdx], term[sepIdx], term[sepIdx+1:]
+				if extractor, ok := qualifiers[qualifier]; ok {
+					switch separator {
+					case ':':
+						matched = (matched && utils.Match(extractor(lib), []string{target}))
+						continue
+					case '=':
+						matched = (matched && strings.ToLower(extractor(lib)) == target)
+						continue
+					}
+				}
+			}
+			// We perform the usual match in the following cases:
+			// 1. Unknown qualifier names revert to basic search terms.
+			// 2. Terms that do not use qv-syntax.
+			matched = (matched && utils.Match(defaultLibraryMatchExtractor(lib), []string{term}))
+		}
+		return matched
+	}
+}

Diff for: commands/lib/search_test.go

@@ -28,6 +28,7 @@ import (
 
 var customIndexPath = paths.New("testdata", "test1")
 var fullIndexPath = paths.New("testdata", "full")
+var qualifiedSearchIndexPath = paths.New("testdata", "qualified_search")
 
 func TestSearchLibrary(t *testing.T) {
 	lm := librariesmanager.NewLibraryManager(customIndexPath, nil)
@@ -94,3 +95,112 @@ func TestSearchLibraryFields(t *testing.T) {
 	require.Len(t, res, 19)
 	require.Equal(t, "FlashStorage", res[0])
 }
+
+func TestSearchLibraryWithQualifiers(t *testing.T) {
+	lm := librariesmanager.NewLibraryManager(qualifiedSearchIndexPath, nil)
+	lm.LoadIndex()
+
+	query := func(q string) []string {
+		libs := []string{}
+		for _, lib := range searchLibrary(&rpc.LibrarySearchRequest{SearchArgs: q}, lm).Libraries {
+			libs = append(libs, lib.Name)
+		}
+		return libs
+	}
+
+	res := query("mesh")
+	require.Len(t, res, 4)
+
+	res = query("name:Mesh")
+	require.Len(t, res, 3)
+
+	res = query("name=Mesh")
+	require.Len(t, res, 0)
+
+	// Space not in double-quoted string
+	res = query("name=Painless Mesh")
+	require.Len(t, res, 0)
+
+	// Embedded space in double-quoted string
+	res = query("name=\"Painless Mesh\"")
+	require.Len(t, res, 1)
+	require.Equal(t, "Painless Mesh", res[0])
+
+	// No closing double-quote - still tokenizes with embedded space
+	res = query("name:\"Painless Mesh")
+	require.Len(t, res, 1)
+
+	// Malformed double-quoted string with escaped first double-quote
+	res = query("name:\\\"Painless Mesh\"")
+	require.Len(t, res, 0)
+
+	res = query("name:mesh author:TMRh20")
+	require.Len(t, res, 1)
+	require.Equal(t, "RF24Mesh", res[0])
+
+	res = query("mesh dependencies:ArduinoJson")
+	require.Len(t, res, 1)
+	require.Equal(t, "Painless Mesh", res[0])
+
+	res = query("architectures:esp author=\"Suraj I.\"")
+	require.Len(t, res, 1)
+	require.Equal(t, "esp8266-framework", res[0])
+
+	res = query("mesh esp")
+	require.Len(t, res, 2)
+
+	res = query("mesh esp paragraph:wifi")
+	require.Len(t, res, 1)
+	require.Equal(t, "esp8266-framework", res[0])
+
+	// Unknown qualifier should revert to original matching
+	res = query("std::array")
+	require.Len(t, res, 1)
+	require.Equal(t, "Array", res[0])
+
+	res = query("data storage")
+	require.Len(t, res, 1)
+	require.Equal(t, "Pushdata_ESP8266_SSL", res[0])
+
+	res = query("category:\"data storage\"")
+	require.Len(t, res, 1)
+	require.Equal(t, "Array", res[0])
+
+	res = query("maintainer:@")
+	require.Len(t, res, 4)
+
+	res = query("sentence:\"A library for NRF24L01(+) devices mesh.\"")
+	require.Len(t, res, 1)
+	require.Equal(t, "RF24Mesh", res[0])
+
+	res = query("types=contributed")
+	require.Len(t, res, 7)
+
+	res = query("version:1.0")
+	require.Len(t, res, 3)
+
+	res = query("version=1.2.1")
+	require.Len(t, res, 1)
+	require.Equal(t, "Array", res[0])
+
+	// Non-SSL URLs
+	res = query("website:http://")
+	require.Len(t, res, 1)
+	require.Equal(t, "RF24Mesh", res[0])
+
+	// Literal double-quote
+	res = query("sentence:\\\"")
+	require.Len(t, res, 1)
+	require.Equal(t, "RTCtime", res[0])
+
+	res = query("license=MIT")
+	require.Len(t, res, 2)
+
+	// Empty string
+	res = query("license=\"\"")
+	require.Len(t, res, 5)
+
+	res = query("provides:painlessmesh.h")
+	require.Len(t, res, 1)
+	require.Equal(t, "Painless Mesh", res[0])
+}