Max open files (#628)

* adds max_open_files flag

* adds doc of max_open_files flag

* amends commmit

Author: chavacava

README.md

@@ -191,7 +191,7 @@ Please notice that if no particular configuration is provided, `revive` will beh
 
 ### Command Line Flags
 
-`revive` accepts three command line parameters:
+`revive` accepts the following command line parameters:
 
 - `-config [PATH]` - path to config file in TOML format, defaults to `$HOME/revive.toml` if present.
 - `-exclude [PATTERN]` - pattern for files/directories/packages to be excluded for linting. You can specify the files you want to exclude for linting either as package name (i.e. `github.com/mgechev/revive`), list them as individual files (i.e. `file.go`), directories (i.e. `./foo/...`), or any combination of the three.
@@ -203,7 +203,10 @@ Please notice that if no particular configuration is provided, `revive` will beh
   - `friendly` - outputs the failures when found. Shows summary of all the failures.
   - `stylish` - formats the failures in a table. Keep in mind that it doesn't stream the output so it might be perceived as slower compared to others.
   - `checkstyle` - outputs the failures in XML format compatible with that of Java's [Checkstyle](https://checkstyle.org/).
-- `-set_exit_status` - set exit status to 1 if any issues are found, overwrites errorCode and warningCode in config.
+- `-max_open_files` -  maximum number of open files at the same time. Defaults to unlimited.
+- `-set_exit_status` - set exit status to 1 if any issues are found, overwrites `errorCode` and `warningCode` in config.
+- `-version` - get revive version.
+
 
 ### Sample Invocations
 

lint/linter.go

@@ -18,12 +18,34 @@ type disabledIntervalsMap = map[string][]DisabledInterval
 
 // Linter is used for linting set of files.
 type Linter struct {
-	reader ReadFile
+	reader         ReadFile
+	fileReadTokens chan struct{}
 }
 
 // New creates a new Linter
-func New(reader ReadFile) Linter {
-	return Linter{reader: reader}
+func New(reader ReadFile, maxOpenFiles int) Linter {
+	var fileReadTokens chan struct{}
+	if maxOpenFiles > 0 {
+		fileReadTokens = make(chan struct{}, maxOpenFiles)
+	}
+	return Linter{
+		reader:         reader,
+		fileReadTokens: fileReadTokens,
+	}
+}
+
+func (l Linter) readFile(path string) (result []byte, err error) {
+	if l.fileReadTokens != nil {
+		// "take" a token by writing to the channel.
+		// It will block if no more space in the channel's buffer
+		l.fileReadTokens <- struct{}{}
+		defer func() {
+			// "free" a token by reading from the channel
+			<-l.fileReadTokens
+		}()
+	}
+
+	return l.reader(path)
 }
 
 var (
@@ -62,7 +84,7 @@ func (l *Linter) lintPackage(filenames []string, ruleSet []Rule, config Config,
 		mu:    sync.Mutex{},
 	}
 	for _, filename := range filenames {
-		content, err := l.reader(filename)
+		content, err := l.readFile(filename)
 		if err != nil {
 			return err
 		}

main.go

@@ -59,7 +59,7 @@ func main() {
 
 	revive := lint.New(func(file string) ([]byte, error) {
 		return ioutil.ReadFile(file)
-	})
+	}, maxOpenFiles)
 
 	lintingRules, err := config.GetLintingRules(conf)
 	if err != nil {
@@ -155,6 +155,7 @@ var (
 	help          bool
 	versionFlag   bool
 	setExitStatus bool
+	maxOpenFiles  int
 )
 
 var originalUsage = flag.Usage
@@ -204,11 +205,12 @@ func init() {
 
 	// command line help strings
 	const (
-		configUsage     = "path to the configuration TOML file, defaults to $HOME/revive.toml, if present (i.e. -config myconf.toml)"
-		excludeUsage    = "list of globs which specify files to be excluded (i.e. -exclude foo/...)"
-		formatterUsage  = "formatter to be used for the output (i.e. -formatter stylish)"
-		versionUsage    = "get revive version"
-		exitStatusUsage = "set exit status to 1 if any issues are found, overwrites errorCode and warningCode in config"
+		configUsage       = "path to the configuration TOML file, defaults to $HOME/revive.toml, if present (i.e. -config myconf.toml)"
+		excludeUsage      = "list of globs which specify files to be excluded (i.e. -exclude foo/...)"
+		formatterUsage    = "formatter to be used for the output (i.e. -formatter stylish)"
+		versionUsage      = "get revive version"
+		exitStatusUsage   = "set exit status to 1 if any issues are found, overwrites errorCode and warningCode in config"
+		maxOpenFilesUsage = "maximum number of open files at the same time"
 	)
 
 	defaultConfigPath := buildDefaultConfigPath()
@@ -218,6 +220,7 @@ func init() {
 	flag.StringVar(&formatterName, "formatter", "", formatterUsage)
 	flag.BoolVar(&versionFlag, "version", false, versionUsage)
 	flag.BoolVar(&setExitStatus, "set_exit_status", false, exitStatusUsage)
+	flag.IntVar(&maxOpenFiles, "max_open_files", 0, maxOpenFilesUsage)
 	flag.Parse()
 
 	// Output build info (version, commit, date and builtBy)

test/utils.go

@@ -44,7 +44,7 @@ func testRule(t *testing.T, filename string, rule lint.Rule, config ...*lint.Rul
 func assertSuccess(t *testing.T, baseDir string, fi os.FileInfo, rules []lint.Rule, config map[string]lint.RuleConfig) error {
 	l := lint.New(func(file string) ([]byte, error) {
 		return ioutil.ReadFile(baseDir + file)
-	})
+	}, 0)
 
 	ps, err := l.Lint([][]string{{fi.Name()}}, rules, lint.Config{
 		Rules: config,
@@ -66,7 +66,7 @@ func assertSuccess(t *testing.T, baseDir string, fi os.FileInfo, rules []lint.Ru
 func assertFailures(t *testing.T, baseDir string, fi os.FileInfo, src []byte, rules []lint.Rule, config map[string]lint.RuleConfig) error {
 	l := lint.New(func(file string) ([]byte, error) {
 		return ioutil.ReadFile(baseDir + file)
-	})
+	}, 0)
 
 	ins := parseInstructions(t, fi.Name(), src)
 	if ins == nil {