all: consolidate invokeGo implementations

Over time we have accumulated a disturbing number of ways to run the Go
command, each of which supported slightly different features and
workarounds. Combine all of them.

This unavoidably introduces some changes in debug logging behavior; I
think we should try to fix them incrementally and consistently.

Updates golang/go#37368.

Change-Id: I664ca8685bf247a226be3cb807789c2fcddf233d
Reviewed-on: https://go-review.googlesource.com/c/tools/+/220737
Run-TryBot: Heschi Kreinick <[email protected]>
Reviewed-by: Rebecca Stambler <[email protected]>

Author: heschi

go/internal/packagesdriver/sizes.go

@@ -11,11 +11,10 @@ import (
 	"encoding/json"
 	"fmt"
 	"go/types"
-	"log"
-	"os"
 	"os/exec"
 	"strings"
-	"time"
+
+	"golang.org/x/tools/internal/gocommand"
 )
 
 var debug = false
@@ -78,97 +77,42 @@ func GetSizes(ctx context.Context, buildFlags, env []string, dir string, usesExp
 }
 
 func GetSizesGolist(ctx context.Context, buildFlags, env []string, dir string, usesExportData bool) (types.Sizes, error) {
-	args := []string{"list", "-f", "{{context.GOARCH}} {{context.Compiler}}"}
-	args = append(args, buildFlags...)
-	args = append(args, "--", "unsafe")
-	stdout, stderr, err := invokeGo(ctx, env, dir, usesExportData, args...)
+	inv := gocommand.Invocation{
+		Verb:       "list",
+		Args:       []string{"-f", "{{context.GOARCH}} {{context.Compiler}}", "--", "unsafe"},
+		Env:        env,
+		BuildFlags: buildFlags,
+		WorkingDir: dir,
+	}
+	stdout, stderr, friendlyErr, rawErr := inv.RunRaw(ctx)
 	var goarch, compiler string
-	if err != nil {
-		if strings.Contains(err.Error(), "cannot find main module") {
+	if rawErr != nil {
+		if strings.Contains(rawErr.Error(), "cannot find main module") {
 			// User's running outside of a module. All bets are off. Get GOARCH and guess compiler is gc.
 			// TODO(matloob): Is this a problem in practice?
-			envout, _, enverr := invokeGo(ctx, env, dir, usesExportData, "env", "GOARCH")
+			inv := gocommand.Invocation{
+				Verb:       "env",
+				Args:       []string{"GOARCH"},
+				Env:        env,
+				WorkingDir: dir,
+			}
+			envout, enverr := inv.Run(ctx)
 			if enverr != nil {
-				return nil, err
+				return nil, enverr
 			}
 			goarch = strings.TrimSpace(envout.String())
 			compiler = "gc"
 		} else {
-			return nil, err
+			return nil, friendlyErr
 		}
 	} else {
 		fields := strings.Fields(stdout.String())
 		if len(fields) < 2 {
-			return nil, fmt.Errorf("could not parse GOARCH and Go compiler in format \"<GOARCH> <compiler>\" from stdout of go command:\n%s\ndir: %s\nstdout: <<%s>>\nstderr: <<%s>>",
-				cmdDebugStr(env, args...), dir, stdout.String(), stderr.String())
+			return nil, fmt.Errorf("could not parse GOARCH and Go compiler in format \"<GOARCH> <compiler>\":\nstdout: <<%s>>\nstderr: <<%s>>",
+				stdout.String(), stderr.String())
 		}
 		goarch = fields[0]
 		compiler = fields[1]
 	}
 	return types.SizesFor(compiler, goarch), nil
 }
-
-// invokeGo returns the stdout and stderr of a go command invocation.
-func invokeGo(ctx context.Context, env []string, dir string, usesExportData bool, args ...string) (*bytes.Buffer, *bytes.Buffer, error) {
-	if debug {
-		defer func(start time.Time) { log.Printf("%s for %v", time.Since(start), cmdDebugStr(env, args...)) }(time.Now())
-	}
-	stdout := new(bytes.Buffer)
-	stderr := new(bytes.Buffer)
-	cmd := exec.CommandContext(ctx, "go", args...)
-	// On darwin the cwd gets resolved to the real path, which breaks anything that
-	// expects the working directory to keep the original path, including the
-	// go command when dealing with modules.
-	// The Go stdlib has a special feature where if the cwd and the PWD are the
-	// same node then it trusts the PWD, so by setting it in the env for the child
-	// process we fix up all the paths returned by the go command.
-	cmd.Env = append(append([]string{}, env...), "PWD="+dir)
-	cmd.Dir = dir
-	cmd.Stdout = stdout
-	cmd.Stderr = stderr
-	if err := cmd.Run(); err != nil {
-		exitErr, ok := err.(*exec.ExitError)
-		if !ok {
-			// Catastrophic error:
-			// - executable not found
-			// - context cancellation
-			return nil, nil, fmt.Errorf("couldn't exec 'go %v': %s %T", args, err, err)
-		}
-
-		// Export mode entails a build.
-		// If that build fails, errors appear on stderr
-		// (despite the -e flag) and the Export field is blank.
-		// Do not fail in that case.
-		if !usesExportData {
-			return nil, nil, fmt.Errorf("go %v: %s: %s", args, exitErr, stderr)
-		}
-	}
-
-	// As of writing, go list -export prints some non-fatal compilation
-	// errors to stderr, even with -e set. We would prefer that it put
-	// them in the Package.Error JSON (see https://golang.org/issue/26319).
-	// In the meantime, there's nowhere good to put them, but they can
-	// be useful for debugging. Print them if $GOPACKAGESPRINTGOLISTERRORS
-	// is set.
-	if len(stderr.Bytes()) != 0 && os.Getenv("GOPACKAGESPRINTGOLISTERRORS") != "" {
-		fmt.Fprintf(os.Stderr, "%s stderr: <<%s>>\n", cmdDebugStr(env, args...), stderr)
-	}
-
-	// debugging
-	if false {
-		fmt.Fprintf(os.Stderr, "%s stdout: <<%s>>\n", cmdDebugStr(env, args...), stdout)
-	}
-
-	return stdout, stderr, nil
-}
-
-func cmdDebugStr(envlist []string, args ...string) string {
-	env := make(map[string]string)
-	for _, kv := range envlist {
-		split := strings.Split(kv, "=")
-		k, v := split[0], split[1]
-		env[k] = v
-	}
-
-	return fmt.Sprintf("GOROOT=%v GOPATH=%v GO111MODULE=%v PWD=%v go %v", env["GOROOT"], env["GOPATH"], env["GO111MODULE"], env["PWD"], args)
-}

go/packages/golist.go

@@ -20,10 +20,10 @@ import (
 	"strconv"
 	"strings"
 	"sync"
-	"time"
 	"unicode"
 
 	"golang.org/x/tools/go/internal/packagesdriver"
+	"golang.org/x/tools/internal/gocommand"
 	"golang.org/x/tools/internal/packagesinternal"
 )
 
@@ -707,29 +707,17 @@ func golistargs(cfg *Config, words []string) []string {
 func (state *golistState) invokeGo(verb string, args ...string) (*bytes.Buffer, error) {
 	cfg := state.cfg
 
-	stdout := new(bytes.Buffer)
-	stderr := new(bytes.Buffer)
-	goArgs := []string{verb}
-	if verb != "env" {
-		goArgs = append(goArgs, cfg.BuildFlags...)
-	}
-	goArgs = append(goArgs, args...)
-	cmd := exec.CommandContext(state.ctx, "go", goArgs...)
-	// On darwin the cwd gets resolved to the real path, which breaks anything that
-	// expects the working directory to keep the original path, including the
-	// go command when dealing with modules.
-	// The Go stdlib has a special feature where if the cwd and the PWD are the
-	// same node then it trusts the PWD, so by setting it in the env for the child
-	// process we fix up all the paths returned by the go command.
-	cmd.Env = append(append([]string{}, cfg.Env...), "PWD="+cfg.Dir)
-	cmd.Dir = cfg.Dir
-	cmd.Stdout = stdout
-	cmd.Stderr = stderr
-	defer func(start time.Time) {
-		cfg.Logf("%s for %v, stderr: <<%s>> stdout: <<%s>>\n", time.Since(start), cmdDebugStr(cmd, goArgs...), stderr, stdout)
-	}(time.Now())
-
-	if err := cmd.Run(); err != nil {
+	inv := &gocommand.Invocation{
+		Verb:       verb,
+		Args:       args,
+		BuildFlags: cfg.BuildFlags,
+		Env:        cfg.Env,
+		Logf:       cfg.Logf,
+		WorkingDir: cfg.Dir,
+	}
+
+	stdout, stderr, _, err := inv.RunRaw(cfg.Context)
+	if err != nil {
 		// Check for 'go' executable not being found.
 		if ee, ok := err.(*exec.Error); ok && ee.Err == exec.ErrNotFound {
 			return nil, fmt.Errorf("'go list' driver requires 'go', but %s", exec.ErrNotFound)
@@ -739,7 +727,7 @@ func (state *golistState) invokeGo(verb string, args ...string) (*bytes.Buffer,
 		if !ok {
 			// Catastrophic error:
 			// - context cancellation
-			return nil, fmt.Errorf("couldn't exec 'go %v': %s %T", args, err, err)
+			return nil, fmt.Errorf("couldn't run 'go': %v", err)
 		}
 
 		// Old go version?
@@ -860,16 +848,6 @@ func (state *golistState) invokeGo(verb string, args ...string) (*bytes.Buffer,
 			return nil, fmt.Errorf("go %v: %s: %s", args, exitErr, stderr)
 		}
 	}
-
-	// As of writing, go list -export prints some non-fatal compilation
-	// errors to stderr, even with -e set. We would prefer that it put
-	// them in the Package.Error JSON (see https://golang.org/issue/26319).
-	// In the meantime, there's nowhere good to put them, but they can
-	// be useful for debugging. Print them if $GOPACKAGESPRINTGOLISTERRORS
-	// is set.
-	if len(stderr.Bytes()) != 0 && os.Getenv("GOPACKAGESPRINTGOLISTERRORS") != "" {
-		fmt.Fprintf(os.Stderr, "%s stderr: <<%s>>\n", cmdDebugStr(cmd, args...), stderr)
-	}
 	return stdout, nil
 }
 

go/packages/packagestest/modules.go

@@ -6,17 +6,16 @@ package packagestest
 
 import (
 	"archive/zip"
-	"bytes"
+	"context"
 	"fmt"
 	"io/ioutil"
 	"os"
-	"os/exec"
 	"path"
 	"path/filepath"
 	"regexp"
 	"strings"
 
-	"golang.org/x/tools/go/packages"
+	"golang.org/x/tools/internal/gocommand"
 )
 
 // Modules is the exporter that produces module layouts.
@@ -170,7 +169,14 @@ func (modules) Finalize(exported *Exported) error {
 
 	// Run go mod download to recreate the mod cache dir with all the extra
 	// stuff in cache. All the files created by Export should be recreated.
-	if err := invokeGo(exported.Config, "mod", "download"); err != nil {
+	inv := gocommand.Invocation{
+		Verb:       "mod",
+		Args:       []string{"download"},
+		Env:        exported.Config.Env,
+		BuildFlags: exported.Config.BuildFlags,
+		WorkingDir: exported.Config.Dir,
+	}
+	if _, err := inv.Run(context.Background()); err != nil {
 		return err
 	}
 	return nil
@@ -237,20 +243,6 @@ func writeModuleProxy(dir, module, ver string, files map[string]string) error {
 	return nil
 }
 
-func invokeGo(cfg *packages.Config, args ...string) error {
-	stdout := new(bytes.Buffer)
-	stderr := new(bytes.Buffer)
-	cmd := exec.Command("go", args...)
-	cmd.Env = append(append([]string{}, cfg.Env...), "PWD="+cfg.Dir)
-	cmd.Dir = cfg.Dir
-	cmd.Stdout = stdout
-	cmd.Stderr = stderr
-	if err := cmd.Run(); err != nil {
-		return fmt.Errorf("go %v: %s: %s", args, err, stderr)
-	}
-	return nil
-}
-
 func modCache(exported *Exported) string {
 	return filepath.Join(exported.temp, "modcache/pkg/mod")
 }

internal/gocommand/invoke.go

@@ -0,0 +1,92 @@
+// Package gocommand is a helper for calling the go command.
+package gocommand
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"os/exec"
+	"strings"
+	"time"
+)
+
+// An Invocation represents a call to the go command.
+type Invocation struct {
+	Verb       string
+	Args       []string
+	BuildFlags []string
+	Env        []string
+	WorkingDir string
+	Logf       func(format string, args ...interface{})
+}
+
+// Run runs the invocation, returning its stdout and an error suitable for
+// human consumption, including stderr.
+func (i *Invocation) Run(ctx context.Context) (*bytes.Buffer, error) {
+	stdout, _, friendly, _ := i.RunRaw(ctx)
+	return stdout, friendly
+}
+
+// RunRaw is like Run, but also returns the raw stderr and error for callers
+// that want to do low-level error handling/recovery.
+func (i *Invocation) RunRaw(ctx context.Context) (stdout *bytes.Buffer, stderr *bytes.Buffer, friendlyError error, rawError error) {
+	log := i.Logf
+	if log == nil {
+		log = func(string, ...interface{}) {}
+	}
+
+	goArgs := []string{i.Verb}
+	switch i.Verb {
+	case "mod":
+		// mod needs the sub-verb before build flags.
+		goArgs = append(goArgs, i.Args[0])
+		goArgs = append(goArgs, i.BuildFlags...)
+		goArgs = append(goArgs, i.Args[1:]...)
+	case "env":
+		// env doesn't take build flags.
+		goArgs = append(goArgs, i.Args...)
+	default:
+		goArgs = append(goArgs, i.BuildFlags...)
+		goArgs = append(goArgs, i.Args...)
+	}
+	cmd := exec.CommandContext(ctx, "go", goArgs...)
+	stdout = &bytes.Buffer{}
+	stderr = &bytes.Buffer{}
+	cmd.Stdout = stdout
+	cmd.Stderr = stderr
+	// On darwin the cwd gets resolved to the real path, which breaks anything that
+	// expects the working directory to keep the original path, including the
+	// go command when dealing with modules.
+	// The Go stdlib has a special feature where if the cwd and the PWD are the
+	// same node then it trusts the PWD, so by setting it in the env for the child
+	// process we fix up all the paths returned by the go command.
+	cmd.Env = append(append([]string{}, i.Env...), "PWD="+i.WorkingDir)
+	cmd.Dir = i.WorkingDir
+
+	defer func(start time.Time) { log("%s for %v", time.Since(start), cmdDebugStr(cmd)) }(time.Now())
+
+	rawError = cmd.Run()
+	friendlyError = rawError
+	if rawError != nil {
+		// Check for 'go' executable not being found.
+		if ee, ok := rawError.(*exec.Error); ok && ee.Err == exec.ErrNotFound {
+			friendlyError = fmt.Errorf("go command required, not found: %v", ee)
+		}
+		if ctx.Err() != nil {
+			friendlyError = ctx.Err()
+		}
+		friendlyError = fmt.Errorf("err: %v: stderr: %s", rawError, stderr)
+	}
+	return
+}
+
+func cmdDebugStr(cmd *exec.Cmd) string {
+	env := make(map[string]string)
+	for _, kv := range cmd.Env {
+		split := strings.Split(kv, "=")
+		k, v := split[0], split[1]
+		env[k] = v
+	}
+
+	return fmt.Sprintf("GOROOT=%v GOPATH=%v GO111MODULE=%v GOPROXY=%v PWD=%v go %v", env["GOROOT"], env["GOPATH"], env["GO111MODULE"], env["GOPROXY"], env["PWD"], cmd.Args)
+}