fix: split large compute file up (#2546)

This is a followup to #2544. That change reduced the size of the compute client from 9.4M to 6.7. This change now splits the compute client into three files ranging from 2.7-1.8M each. This should allow editors and tooling to better be able to process the client.

Author: codyoss

compute/v0.alpha/compute-gen.go

@@ -32,6 +32,11 @@ const (
 	googleDiscoveryURL        = "https://www.googleapis.com/discovery/v1/apis"
 	googleDefaultUniverse     = "googleapis.com"
 	universeDomainPlaceholder = "UNIVERSE_DOMAIN"
+
+	splitFileSeperator = `// =*=*"`
+	// Size the buffer so there is room for the header and contents when
+	// splitting up large files.
+	splitFileHeaderSize = 500
 )
 
 var (
@@ -75,6 +80,10 @@ var skipAPIGeneration = map[string]bool{
 	"datalineage:v1":       true,
 }
 
+var apisToSplit = map[string]bool{
+	"compute": true,
+}
+
 // API represents an API to generate, as well as its state while it's
 // generating.
 type API struct {
@@ -226,7 +235,7 @@ func getAPIs() []*API {
 	} else {
 		bytes = slurpURL(*apisURL)
 		if *publicOnly {
-			if err := writeFile(apiListFile, bytes); err != nil {
+			if err := writeFile(apiListFile, "", bytes); err != nil {
 				log.Fatal(err)
 			}
 		}
@@ -300,7 +309,7 @@ func apiFromFile(file string) (*API, error) {
 func checkAndUpdateSpecFile(file string, contents []byte) error {
 	// check if file exists
 	if _, err := os.Stat(file); os.IsNotExist(err) {
-		return writeFile(file, contents)
+		return writeFile(file, "", contents)
 	}
 	existing, err := os.ReadFile(file)
 	if err != nil {
@@ -309,7 +318,7 @@ func checkAndUpdateSpecFile(file string, contents []byte) error {
 	if err := isNewerRevision(existing, contents); err != nil {
 		return err
 	}
-	return writeFile(file, contents)
+	return writeFile(file, "", contents)
 }
 
 // isNewerRevision returns nil if the contents of new has a newer revision than
@@ -331,7 +340,7 @@ func isNewerRevision(old []byte, new []byte) error {
 	return nil
 }
 
-func writeFile(file string, contents []byte) error {
+func writeFile(file, pkg string, contents []byte) error {
 	// Don't write it if the contents are identical.
 	existing, err := os.ReadFile(file)
 	if err == nil && (bytes.Equal(existing, contents) || basicallyEqual(existing, contents)) {
@@ -341,7 +350,49 @@ func writeFile(file string, contents []byte) error {
 	if err = os.MkdirAll(outdir, 0755); err != nil {
 		return fmt.Errorf("failed to Mkdir %s: %v", outdir, err)
 	}
-	return os.WriteFile(file, contents, 0644)
+	// Don't try to split spec files, json or non-allowlisted packages
+	if pkg == "" || !apisToSplit[pkg] {
+		return os.WriteFile(file, contents, 0644)
+	}
+
+	// Split generated file out into multiple
+	bs := bytes.Split(contents, []byte(splitFileSeperator))
+	for i, b := range bs {
+		var name string
+		var newB []byte
+		if i == 0 {
+			// For the base case, use the provided inputs as is
+			name = file
+			var err error
+			newB, err = format.Source(b)
+			if err != nil {
+				return err
+			}
+		} else {
+			// determine the new file name
+			base := filepath.Dir(file)
+			fileNum := i + 1
+			name = filepath.Join(base, fmt.Sprintf("%s%d-gen.go", pkg, fileNum))
+
+			// prepend file header, package, and imports
+			var buf bytes.Buffer
+			// Size the buffer so there is room for the header and contents
+			buf.Grow(len(b) + splitFileHeaderSize)
+			splitFileHeading(&buf, pkg)
+			_, err := buf.Write(b)
+			if err != nil {
+				return err
+			}
+			newB, err = format.Source(buf.Bytes())
+			if err != nil {
+				return err
+			}
+		}
+		if err := os.WriteFile(name, newB, 0644); err != nil {
+			return err
+		}
+	}
+	return nil
 }
 
 var ignoreLines = regexp.MustCompile(`(?m)^\s+"(?:etag|revision)": ".+\n`)
@@ -589,7 +640,7 @@ func (a *API) WriteGeneratedCode() error {
 	}
 
 	code, err := a.GenerateCode()
-	errw := writeFile(genfilename, code)
+	errw := writeFile(genfilename, a.Package(), code)
 	if err == nil {
 		err = errw
 	}
@@ -862,8 +913,12 @@ func (a *API) GenerateCode() ([]byte, error) {
 	for _, meth := range a.APIMethods() {
 		meth.generateCode()
 	}
-
-	for _, res := range a.doc.Resources {
+	a.insertSplitFileComment()
+	rCnt := len(a.doc.Resources) / 2
+	for i, res := range a.doc.Resources {
+		if i == rCnt {
+			a.insertSplitFileComment()
+		}
 		a.generateResourceMethods(res)
 	}
 
@@ -874,6 +929,55 @@ func (a *API) GenerateCode() ([]byte, error) {
 	return clean, nil
 }
 
+func (a *API) insertSplitFileComment() {
+	if apisToSplit[a.Package()] {
+		a.pn("")
+		a.pn(splitFileSeperator)
+		a.pn("")
+	}
+}
+
+// splitFileHeading writes the file preamble used when generating a split file
+// client like compute.
+func splitFileHeading(w io.Writer, pkg string) {
+	pn := func(format string, args ...interface{}) {
+		_, err := fmt.Fprintf(w, format+"\n", args...)
+		if err != nil {
+			panic(err)
+		}
+	}
+
+	pn("// Copyright %s Google LLC.", *copyrightYear)
+	pn("// Use of this source code is governed by a BSD-style")
+	pn("// license that can be found in the LICENSE file.")
+	pn("")
+	pn("// Code generated file. DO NOT EDIT.")
+	pn("")
+	pn("package %s", pkg)
+	pn("")
+	pn("import (")
+	for _, imp := range []string{
+		"context",
+		"fmt",
+		"io",
+		"net/http",
+	} {
+		pn("  %q", imp)
+	}
+	pn("")
+	for _, imp := range []struct {
+		pkg   string
+		lname string
+	}{
+		{*gensupportPkg, "gensupport"},
+		{*googleapiPkg, "googleapi"},
+	} {
+		pn("  %s %q", imp.lname, imp.pkg)
+	}
+	pn(")")
+	pn("")
+}
+
 func (a *API) generateScopeConstants() {
 	scopes := a.doc.Auth.OAuth2Scopes
 	if len(scopes) == 0 {