helper/resource: Support TestStep provider handling (#972)

Reference: #253
Reference: #628
Reference: #779

Provider developers can now select whether to configure providers for acceptance testing at the `TestCase` or `TestStep` level. Only one level may be used in this current implementation, however it may be possible to allow merged `TestCase` and `TestStep` configuration with additional validation logic to ensure a single provider is not specified multiple times across the merge result of all fields.

This change also introduces some upfront `TestCase` and `TestStep` configuration validation when calling any of the `Test` functions, failing the test early if a problem is detected. There are other validations that are possible, however these are considered out of scope.

Author: bflad

.changelog/972.txt

@@ -0,0 +1,11 @@
+```release-note:note
+helper/resource: Provider references or external installation can now be handled at either the `TestCase` or `TestStep` level. Using the `TestStep` handling, advanced use cases are now enabled such as state upgrade acceptance testing.
+```
+
+```release-note:enhancement
+helper/resource: Added `TestStep` type `ExternalProviders`, `ProtoV5ProviderFactories`, `ProtoV6ProviderFactories`, and `ProviderFactories` fields
+```
+
+```release-note:bug
+helper/resource: Removed extraneous `terraform state show` command when not using the `TestStep` type `Taint` field
+```

helper/resource/plugin.go

@@ -19,17 +19,108 @@ import (
 	testing "github.com/mitchellh/go-testing-interface"
 )
 
+// protov5ProviderFactory is a function which is called to start a protocol
+// version 5 provider server.
+type protov5ProviderFactory func() (tfprotov5.ProviderServer, error)
+
+// protov5ProviderFactories is a mapping of provider addresses to provider
+// factory for protocol version 5 provider servers.
+type protov5ProviderFactories map[string]func() (tfprotov5.ProviderServer, error)
+
+// merge combines provider factories.
+//
+// In case of an overlapping entry, the later entry will overwrite the previous
+// value.
+func (pf protov5ProviderFactories) merge(otherPfs ...protov5ProviderFactories) protov5ProviderFactories {
+	result := make(protov5ProviderFactories)
+
+	for name, providerFactory := range pf {
+		result[name] = providerFactory
+	}
+
+	for _, otherPf := range otherPfs {
+		for name, providerFactory := range otherPf {
+			result[name] = providerFactory
+		}
+	}
+
+	return result
+}
+
+// protov6ProviderFactory is a function which is called to start a protocol
+// version 6 provider server.
+type protov6ProviderFactory func() (tfprotov6.ProviderServer, error)
+
+// protov6ProviderFactories is a mapping of provider addresses to provider
+// factory for protocol version 6 provider servers.
+type protov6ProviderFactories map[string]func() (tfprotov6.ProviderServer, error)
+
+// merge combines provider factories.
+//
+// In case of an overlapping entry, the later entry will overwrite the previous
+// value.
+func (pf protov6ProviderFactories) merge(otherPfs ...protov6ProviderFactories) protov6ProviderFactories {
+	result := make(protov6ProviderFactories)
+
+	for name, providerFactory := range pf {
+		result[name] = providerFactory
+	}
+
+	for _, otherPf := range otherPfs {
+		for name, providerFactory := range otherPf {
+			result[name] = providerFactory
+		}
+	}
+
+	return result
+}
+
+// sdkProviderFactory is a function which is called to start a SDK provider
+// server.
+type sdkProviderFactory func() (*schema.Provider, error)
+
+// protov6ProviderFactories is a mapping of provider addresses to provider
+// factory for protocol version 6 provider servers.
+type sdkProviderFactories map[string]func() (*schema.Provider, error)
+
+// merge combines provider factories.
+//
+// In case of an overlapping entry, the later entry will overwrite the previous
+// value.
+func (pf sdkProviderFactories) merge(otherPfs ...sdkProviderFactories) sdkProviderFactories {
+	result := make(sdkProviderFactories)
+
+	for name, providerFactory := range pf {
+		result[name] = providerFactory
+	}
+
+	for _, otherPf := range otherPfs {
+		for name, providerFactory := range otherPf {
+			result[name] = providerFactory
+		}
+	}
+
+	return result
+}
+
 type providerFactories struct {
-	legacy  map[string]func() (*schema.Provider, error)
-	protov5 map[string]func() (tfprotov5.ProviderServer, error)
-	protov6 map[string]func() (tfprotov6.ProviderServer, error)
+	legacy  sdkProviderFactories
+	protov5 protov5ProviderFactories
+	protov6 protov6ProviderFactories
 }
 
-func runProviderCommand(ctx context.Context, t testing.T, f func() error, wd *plugintest.WorkingDir, factories providerFactories) error {
+func runProviderCommand(ctx context.Context, t testing.T, f func() error, wd *plugintest.WorkingDir, factories *providerFactories) error {
 	// don't point to this as a test failure location
 	// point to whatever called it
 	t.Helper()
 
+	// This should not happen, but prevent panics just in case.
+	if factories == nil {
+		err := fmt.Errorf("Provider factories are missing to run Terraform command. Please report this bug in the testing framework.")
+		logging.HelperResourceError(ctx, err.Error())
+		return err
+	}
+
 	// Run the providers in the same process as the test runner using the
 	// reattach behavior in Terraform. This ensures we get test coverage
 	// and enables the use of delve as a debugger.

helper/resource/plugin_test.go

@@ -0,0 +1,236 @@
+package resource
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/hashicorp/terraform-plugin-go/tfprotov5"
+	"github.com/hashicorp/terraform-plugin-go/tfprotov6"
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
+)
+
+func TestProtoV5ProviderFactoriesMerge(t *testing.T) {
+	t.Parallel()
+
+	testProviderFactory1 := func() (tfprotov5.ProviderServer, error) {
+		return nil, nil
+	}
+	testProviderFactory2 := func() (tfprotov5.ProviderServer, error) {
+		return nil, nil
+	}
+
+	// Function pointers do not play well with go-cmp, so convert these
+	// into their stringified address for comparison.
+	transformer := cmp.Transformer(
+		"protov5ProviderFactory",
+		func(pf protov5ProviderFactory) string {
+			return fmt.Sprintf("%v", pf)
+		},
+	)
+
+	testCases := map[string]struct {
+		pf       protov5ProviderFactories
+		others   []protov5ProviderFactories
+		expected protov5ProviderFactories
+	}{
+		"no-overlap": {
+			pf: protov5ProviderFactories{
+				"test1": testProviderFactory1,
+			},
+			others: []protov5ProviderFactories{
+				{
+					"test2": testProviderFactory1,
+				},
+				{
+					"test3": testProviderFactory1,
+				},
+			},
+			expected: protov5ProviderFactories{
+				"test1": testProviderFactory1,
+				"test2": testProviderFactory1,
+				"test3": testProviderFactory1,
+			},
+		},
+		"overlap": {
+			pf: protov5ProviderFactories{
+				"test": testProviderFactory1,
+			},
+			others: []protov5ProviderFactories{
+				{
+					"test": testProviderFactory1,
+				},
+				{
+					"test": testProviderFactory2,
+				},
+			},
+			expected: protov5ProviderFactories{
+				"test": testProviderFactory2,
+			},
+		},
+	}
+
+	for name, testCase := range testCases {
+		name, testCase := name, testCase
+
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			got := testCase.pf.merge(testCase.others...)
+
+			if diff := cmp.Diff(got, testCase.expected, transformer); diff != "" {
+				t.Errorf("unexpected difference: %s", diff)
+			}
+		})
+	}
+}
+
+func TestProtoV6ProviderFactoriesMerge(t *testing.T) {
+	t.Parallel()
+
+	testProviderFactory1 := func() (tfprotov6.ProviderServer, error) {
+		return nil, nil
+	}
+	testProviderFactory2 := func() (tfprotov6.ProviderServer, error) {
+		return nil, nil
+	}
+
+	// Function pointers do not play well with go-cmp, so convert these
+	// into their stringified address for comparison.
+	transformer := cmp.Transformer(
+		"protov6ProviderFactory",
+		func(pf protov6ProviderFactory) string {
+			return fmt.Sprintf("%v", pf)
+		},
+	)
+
+	testCases := map[string]struct {
+		pf       protov6ProviderFactories
+		others   []protov6ProviderFactories
+		expected protov6ProviderFactories
+	}{
+		"no-overlap": {
+			pf: protov6ProviderFactories{
+				"test1": testProviderFactory1,
+			},
+			others: []protov6ProviderFactories{
+				{
+					"test2": testProviderFactory1,
+				},
+				{
+					"test3": testProviderFactory1,
+				},
+			},
+			expected: protov6ProviderFactories{
+				"test1": testProviderFactory1,
+				"test2": testProviderFactory1,
+				"test3": testProviderFactory1,
+			},
+		},
+		"overlap": {
+			pf: protov6ProviderFactories{
+				"test": testProviderFactory1,
+			},
+			others: []protov6ProviderFactories{
+				{
+					"test": testProviderFactory1,
+				},
+				{
+					"test": testProviderFactory2,
+				},
+			},
+			expected: protov6ProviderFactories{
+				"test": testProviderFactory2,
+			},
+		},
+	}
+
+	for name, testCase := range testCases {
+		name, testCase := name, testCase
+
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			got := testCase.pf.merge(testCase.others...)
+
+			if diff := cmp.Diff(got, testCase.expected, transformer); diff != "" {
+				t.Errorf("unexpected difference: %s", diff)
+			}
+		})
+	}
+}
+
+func TestSdkProviderFactoriesMerge(t *testing.T) {
+	t.Parallel()
+
+	testProviderFactory1 := func() (*schema.Provider, error) {
+		return nil, nil
+	}
+	testProviderFactory2 := func() (*schema.Provider, error) {
+		return nil, nil
+	}
+
+	// Function pointers do not play well with go-cmp, so convert these
+	// into their stringified address for comparison.
+	transformer := cmp.Transformer(
+		"sdkProviderFactory",
+		func(pf sdkProviderFactory) string {
+			return fmt.Sprintf("%v", pf)
+		},
+	)
+
+	testCases := map[string]struct {
+		pf       sdkProviderFactories
+		others   []sdkProviderFactories
+		expected sdkProviderFactories
+	}{
+		"no-overlap": {
+			pf: sdkProviderFactories{
+				"test1": testProviderFactory1,
+			},
+			others: []sdkProviderFactories{
+				{
+					"test2": testProviderFactory1,
+				},
+				{
+					"test3": testProviderFactory1,
+				},
+			},
+			expected: sdkProviderFactories{
+				"test1": testProviderFactory1,
+				"test2": testProviderFactory1,
+				"test3": testProviderFactory1,
+			},
+		},
+		"overlap": {
+			pf: sdkProviderFactories{
+				"test": testProviderFactory1,
+			},
+			others: []sdkProviderFactories{
+				{
+					"test": testProviderFactory1,
+				},
+				{
+					"test": testProviderFactory2,
+				},
+			},
+			expected: sdkProviderFactories{
+				"test": testProviderFactory2,
+			},
+		},
+	}
+
+	for name, testCase := range testCases {
+		name, testCase := name, testCase
+
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			got := testCase.pf.merge(testCase.others...)
+
+			if diff := cmp.Diff(got, testCase.expected, transformer); diff != "" {
+				t.Errorf("unexpected difference: %s", diff)
+			}
+		})
+	}
+}