tke支持cluster_version升级时携带历史cluster_extra_args记录

CHANGELOG.md

@@ -1,4 +1,13 @@
-## 1.56.2 (Unreleased)
+## 1.56.3 (Unreleased)
+## 1.56.2 (April 19, 2021)
+BUG FIXES:
+
+* Remove `ResourceInsufficient` from `retryableErrorCode`.
+
+ENHANCEMENTS:
+
+* Resource: `tencentcloud_kubernetes_cluster` upgrade `cluster_version` will send old `cluster_extra_args` to tke.
+
 ## 1.56.1 (April 6，2021)
 
 BUG FIXES:

go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/mattn/go-colorable v0.1.6 // indirect
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/pkg/errors v0.9.1
-	github.com/tencentcloud/tencentcloud-sdk-go v1.0.123
+	github.com/tencentcloud/tencentcloud-sdk-go v1.0.137
 	github.com/yangwenmai/ratelimit v0.0.0-20180104140304-44221c2292e1
 	github.com/zclconf/go-cty v1.4.2 // indirect
 	golang.org/x/sys v0.0.0-20200523222454-059865788121 // indirect

go.sum

@@ -443,6 +443,8 @@ github.com/tdakkota/asciicheck v0.0.0-20200416190851-d7f85be797a2 h1:Xr9gkxfOP0K
 github.com/tdakkota/asciicheck v0.0.0-20200416190851-d7f85be797a2/go.mod h1:yHp0ai0Z9gUljN3o0xMhYJnH/IcvkdTBOX2fmJ93JEM=
 github.com/tencentcloud/tencentcloud-sdk-go v1.0.123 h1:6yOMrmTuKtW1Y8VtWgN5Jt3jUcWnWtRX0gZfD1LxRcc=
 github.com/tencentcloud/tencentcloud-sdk-go v1.0.123/go.mod h1:asUz5BPXxgoPGaRgZaVm1iGcUAuHyYUo1nXqKa83cvI=
+github.com/tencentcloud/tencentcloud-sdk-go v1.0.137 h1:NEV2Dp7NofwDsyT0OpY5W0fEqsgsbKETD5gWI7/Nn/s=
+github.com/tencentcloud/tencentcloud-sdk-go v1.0.137/go.mod h1:asUz5BPXxgoPGaRgZaVm1iGcUAuHyYUo1nXqKa83cvI=
 github.com/tetafro/godot v0.3.7 h1:+mecr7RKrUKB5UQ1gwqEMn13sDKTyDR8KNIquB9mm+8=
 github.com/tetafro/godot v0.3.7/go.mod h1:/7NLHhv08H1+8DNj0MElpAACw1ajsCuf3TKNQxA5S+0=
 github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e h1:RumXZ56IrCj4CL+g1b9OL/oH0QnsF976bC8xQFYUD5Q=

k8s.tf

@@ -0,0 +1,98 @@
+variable "availability_zone_first" {
+  default = "ap-guangzhou-4"
+}
+
+variable "availability_zone_second" {
+  default = "ap-guangzhou-4"
+}
+
+variable "cluster_cidr" {
+  default = "10.31.0.0/20"
+}
+
+variable "default_instance_type" {
+  default = "S2.MEDIUM2"
+  #default = "SA2.2XLARGE16"
+}
+
+data "tencentcloud_vpc_subnets" "vpc_first" {
+  is_default        = true
+  availability_zone = var.availability_zone_first
+}
+
+data "tencentcloud_vpc_subnets" "vpc_second" {
+  is_default        = true
+  availability_zone = var.availability_zone_second
+}
+
+resource "tencentcloud_kubernetes_cluster" "managed_cluster" {
+  vpc_id                                     = data.tencentcloud_vpc_subnets.vpc_first.instance_list.0.vpc_id
+  cluster_cidr                               = var.cluster_cidr
+  cluster_max_pod_num                        = 32
+  cluster_name                               = "rosta-test"
+  cluster_desc                               = "test cluster desc"
+  cluster_max_service_num                    = 32
+  cluster_internet                           = true
+  managed_cluster_internet_security_policies = ["3.3.3.3", "1.1.1.1"]
+  cluster_deploy_type                        = "MANAGED_CLUSTER"
+  cluster_version                            = "1.16.3"
+  #cluster_version                            = "1.18.4"
+
+  cluster_extra_args {
+    kube_apiserver = [
+      "service-account-issuer=kubernetes.default.svc", 
+      "service-account-signing-key-file=/etc/kubernetes/files/apiserver/service-account.key",
+      "api-audiences=kubernetes.default.svc"
+    ]
+  }
+
+  worker_config {
+    count                      = 1
+    availability_zone          = var.availability_zone_first
+    instance_type              = var.default_instance_type
+    system_disk_type           = "CLOUD_SSD"
+    system_disk_size           = 60
+    internet_charge_type       = "TRAFFIC_POSTPAID_BY_HOUR"
+    internet_max_bandwidth_out = 100
+    public_ip_assigned         = true
+    subnet_id                  = data.tencentcloud_vpc_subnets.vpc_first.instance_list.0.subnet_id
+
+    data_disk {
+      disk_type = "CLOUD_PREMIUM"
+      disk_size = 50
+    }
+
+    enhanced_security_service = false
+    enhanced_monitor_service  = false
+    user_data                 = "dGVzdA=="
+    password                  = "ZZXXccvv1212"
+  }
+
+  worker_config {
+    count                      = 1
+    availability_zone          = var.availability_zone_second
+    instance_type              = var.default_instance_type
+    system_disk_type           = "CLOUD_SSD"
+    system_disk_size           = 60
+    internet_charge_type       = "TRAFFIC_POSTPAID_BY_HOUR"
+    internet_max_bandwidth_out = 100
+    public_ip_assigned         = true
+    subnet_id                  = data.tencentcloud_vpc_subnets.vpc_second.instance_list.0.subnet_id
+
+    data_disk {
+      disk_type = "CLOUD_PREMIUM"
+      disk_size = 50
+    }
+
+    enhanced_security_service = false
+    enhanced_monitor_service  = false
+    user_data                 = "dGVzdA=="
+    password                  = "ZZXXccvv1212"
+  cam_role_name       = "CVM_QcsRole"
+  }
+
+  labels = {
+    "test1" = "test1",
+    "test2" = "test2",
+  }
+}

provider.tf

@@ -0,0 +1,13 @@
+terraform {
+  required_providers {
+    tencentcloud = {
+      source = "tencentcloudstack/tencentcloud"
+#      version = "1.53.0"
+    }
+  }
+}
+
+provider "tencentcloud" {
+  region = "ap-guangzhou"
+}
+

tencentcloud/common.go

@@ -50,7 +50,7 @@ var retryableErrorCode = []string{
 	"TradeUnknownError",
 	"RequestLimitExceeded",
 	"ResourceInUse",
-	"ResourceInsufficient",
+	//"ResourceInsufficient",
 	"ResourceUnavailable",
 	// cbs
 	"ResourceBusy",

tencentcloud/resource_tc_kubernetes_cluster.go

@@ -2124,9 +2124,12 @@ func resourceTencentCloudTkeClusterUpdate(d *schema.ResourceData, meta interface
 		if !isOk {
 			return fmt.Errorf("version %s is unsupported", newVersion)
 		}
-
+		extraArgs, ok := d.GetOk("cluster_extra_args")
+		if !ok {
+			extraArgs = nil
+		}
 		err = resource.Retry(writeRetryTimeout, func() *resource.RetryError {
-			inErr := tkeService.ModifyClusterVersion(ctx, id, newVersion)
+			inErr := tkeService.ModifyClusterVersion(ctx, id, newVersion, extraArgs)
 			if inErr != nil {
 				return retryError(inErr)
 			}

tencentcloud/service_tencentcloud_tke.go

@@ -825,14 +825,52 @@ func (me *TkeService) ModifyClusterAttribute(ctx context.Context, id string, pro
 	return
 }
 
-func (me *TkeService) ModifyClusterVersion(ctx context.Context, id string, clusterVersion string) (errRet error) {
+func (me *TkeService) ModifyClusterVersion(ctx context.Context, id string, clusterVersion string, extraArgs interface{}) (errRet error) {
 	logId := getLogId(ctx)
 	request := tke.NewUpdateClusterVersionRequest()
 	defer func() {
 		if errRet != nil {
 			log.Printf("[CRITAL]%s api[%s] fail, reason[%s]\n", logId, request.GetAction(), errRet.Error())
 		}
 	}()
+
+	if extraArgs != nil && len(extraArgs.([]interface{})) > 0 {
+		// the first elem is in use
+		extraInterface := extraArgs.([]interface{})
+		extraMap := extraInterface[0].(map[string]interface{})
+
+		kas := make([]*string, 0)
+		if kaArgs, exist := extraMap["kube_apiserver"]; exist {
+			args := kaArgs.([]interface{})
+			for index := range args {
+				str := args[index].(string)
+				kas = append(kas, &str)
+			}
+		}
+		kcms := make([]*string, 0)
+		if kcmArgs, exist := extraMap["kube_controller_manager"]; exist {
+			args := kcmArgs.([]interface{})
+			for index := range args {
+				str := args[index].(string)
+				kcms = append(kcms, &str)
+			}
+		}
+		kss := make([]*string, 0)
+		if ksArgs, exist := extraMap["kube_scheduler"]; exist {
+			args := ksArgs.([]interface{})
+			for index := range args {
+				str := args[index].(string)
+				kss = append(kss, &str)
+			}
+		}
+
+		request.ExtraArgs = &tke.ClusterExtraArgs{
+			KubeAPIServer: kas,
+			KubeControllerManager: kcms,
+			KubeScheduler: kss,
+		}
+	}
+
 	request.ClusterId = &id
 	request.DstVersion = &clusterVersion