resource scanning added to obtain creator code (#2519)

* resource scanning added to obtain creator code

* add get resource creator account info code

* add resource create action

* update query rule

* update create time parse code

Author: bruceybian

go.mod

@@ -46,7 +46,7 @@ require (
 	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/clb v1.0.847
 	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cloudaudit v1.0.544
 	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cls v1.0.711
-	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.847
+	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.856
 	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cvm v1.0.624
 	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cwp v1.0.762
 	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cynosdb v1.0.692
@@ -110,6 +110,7 @@ require (
 	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cdwpg v1.0.772
 	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dasb v1.0.798
 	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/oceanus v1.0.831
+	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/billing v1.0.856
 	github.com/wI2L/jsondiff v0.3.0
 	k8s.io/apimachinery v0.22.4
 )

go.sum

@@ -834,6 +834,8 @@ github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/bi v1.0.770 h1:5FWt9F+a
 github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/bi v1.0.770/go.mod h1:zuWcAJVIXbHbOR7i6SKK0wNlvPLt5Su4ijOZGMJbUi8=
 github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/bi v1.0.824 h1:DVKvZ6h+qd7tadUrCjVAkCCmE3TsbK2ZmwGd3AJcpWc=
 github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/bi v1.0.824/go.mod h1:DvBpDX/qdJG4KKLeULmRvhAjPYiw8za0HeTSu2y/lFw=
+github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/billing v1.0.856 h1:zJBb69FBBNZko4PbDScqj9wogPatzNYEXAJ862IBoMQ=
+github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/billing v1.0.856/go.mod h1:P+VGibeVh8pW4N85x9ee0xzXsZoT9jzjHYD8GaAiyG4=
 github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cam v1.0.760 h1:Ky9dRsTL2HXKWUrTFpQFZWQ1TrM+o+P35kczR7thalo=
 github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cam v1.0.760/go.mod h1:AAfdrxknvUedvigxbbzKQLxN+1EG5NPbytpiqmfuFvU=
 github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cat v1.0.760 h1:oGY4IigfIw0iQKh3/cOY29KBeEeFbvJft69e0beyfdI=
@@ -954,6 +956,8 @@ github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.845 h1:fiiV
 github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.845/go.mod h1:r5r4xbfxSaeR04b166HGsBa/R4U3SueirEUpXGuw+Q0=
 github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.847 h1:ITZmxAWfbr5yikJ4T30yVYMW3jpa/oTmNbPnw/h1Vq0=
 github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.847/go.mod h1:r5r4xbfxSaeR04b166HGsBa/R4U3SueirEUpXGuw+Q0=
+github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.856 h1:4PaaKxPjh0dmRzzz0CRZSZlbvzQIPcg4TE1Ibz0Cdlk=
+github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.856/go.mod h1:r5r4xbfxSaeR04b166HGsBa/R4U3SueirEUpXGuw+Q0=
 github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cvm v1.0.624 h1:nEZqsoqt1pEoaP9JjkHQy3/H00suCfzlHW1qOm2nYD8=
 github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cvm v1.0.624/go.mod h1:+TXSVyeKwt1IhZRqKPbTREteBcP+K07Q846/ilNzLWA=
 github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cwp v1.0.762 h1:2egy69SP/wPsmnfozcQVZ6tUY6F6N/TpEe/7xtXrc/8=

tencentcloud/common/cloud_common.go

@@ -0,0 +1,184 @@
+package common
+
+import (
+	"encoding/json"
+	cls "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cls/v20201016"
+	"log"
+	"strconv"
+	"strings"
+	"time"
+
+	cam "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cam/v20190116"
+	"github.com/tencentcloudstack/terraform-provider-tencentcloud/tencentcloud/connectivity"
+	"github.com/tencentcloudstack/terraform-provider-tencentcloud/tencentcloud/internal/helper"
+)
+
+const (
+	// DefaultSearchLogStartTimestamp sync logs start time 2023-11-07 16:41:00
+	DefaultSearchLogStartTimestamp = 1699346460000
+
+	DefaultTopicId = "aef50d54-b17d-4782-8618-a7873203ec29"
+
+	QueryGrammarRule = " AND "
+)
+
+// ResourceAccountInfo 资源账户信息
+type ResourceAccountInfo struct {
+	ResourceType string // 资源类型
+	ResourceName string // 资源名称
+	AccountId    string // 主账号ID
+	PrincipalId  string // 用户ID
+	UserName     string // 用户名
+}
+
+// GetResourceCreatorAccountInfo get resource creator user info
+func GetResourceCreatorAccountInfo(client *connectivity.TencentCloudClient, resourceCreateAction string, resources []*ResourceInstance) map[string]*ResourceAccountInfo {
+	resourceIdToSubAccountInfoMap := make(map[string]*ResourceAccountInfo)
+	if resourceCreateAction == "" {
+		return resourceIdToSubAccountInfoMap
+	}
+
+	request := cls.NewSearchLogRequest()
+	request.From = helper.IntInt64(DefaultSearchLogStartTimestamp)
+	request.To = helper.Int64(CurrentTimeMillisecond())
+	request.TopicId = helper.String(DefaultTopicId)
+
+	for _, r := range resources {
+		query := resourceCreateAction + QueryGrammarRule
+		if r.Id != "" {
+			query = query + r.Id
+		} else if r.Name != "" {
+			query = query + r.Name
+		} else {
+			continue
+		}
+		request.Query = helper.String(query)
+
+		response, err := client.UseClsClient().SearchLog(request)
+		if err != nil {
+			log.Printf("[CRITAL] search resource[%v] log data error: %v", r.Id, err.Error())
+			return resourceIdToSubAccountInfoMap
+		}
+		if response == nil || response.Response == nil {
+			log.Printf("[CRITAL] search resource[%v] log data response is nil", r.Id)
+			return resourceIdToSubAccountInfoMap
+		}
+		if len(response.Response.Results) == 0 {
+			log.Printf("[CRITAL] search resource[%v] log data response results is empty", r.Id)
+			return resourceIdToSubAccountInfoMap
+		}
+
+		result := response.Response.Results[0]
+		if result != nil {
+			var jsonData string
+			if len(*result.LogJson) > 2 {
+				jsonData = *result.LogJson
+			} else if len(*result.RawLog) > 2 {
+				jsonData = *result.RawLog
+			} else {
+				continue
+			}
+
+			resourceAccountInfo := ParseLogJsonData(jsonData)
+			if resourceAccountInfo.PrincipalId == resourceAccountInfo.UserName &&
+				resourceAccountInfo.PrincipalId != resourceAccountInfo.AccountId {
+				userName := GetSubAccountUserName(client, resourceAccountInfo.PrincipalId)
+				resourceAccountInfo.UserName = userName
+			}
+			resourceIdToSubAccountInfoMap[r.Id] = resourceAccountInfo
+		}
+	}
+
+	return resourceIdToSubAccountInfoMap
+}
+
+// GetSubAccountUserName get sub account user name
+func GetSubAccountUserName(client *connectivity.TencentCloudClient, uin string) string {
+	uinNum, err := strconv.ParseUint(uin, 10, 64)
+	if err != nil {
+		log.Printf("[CRITAL] parse uin[%v] to uint64 type error: %v", uin, err.Error())
+		return ""
+	}
+
+	request := cam.NewDescribeSubAccountsRequest()
+
+	uinArray := []*uint64{helper.Uint64(uinNum)}
+	request.FilterSubAccountUin = uinArray
+
+	response, err := client.UseCamClient().DescribeSubAccounts(request)
+	if err != nil {
+		log.Printf("[CRITAL] get sub account[%v] data error: %v", uin, err.Error())
+		return ""
+	}
+	if response == nil || response.Response == nil {
+		log.Printf("[CRITAL] get sub account[%v] data response is nil", uin)
+		return ""
+	}
+
+	name := response.Response.SubAccounts[0].Name
+	return *name
+}
+
+// CurrentTimeMillisecond get the current millisecond timestamp
+func CurrentTimeMillisecond() int64 {
+	return time.Now().UnixNano() / int64(time.Millisecond)
+}
+
+func ParseLogJsonData(jsonData string) *ResourceAccountInfo {
+	if jsonData == "" {
+		return nil
+	}
+
+	var data map[string]interface{}
+	err := json.Unmarshal([]byte(jsonData), &data)
+	if err != nil {
+		log.Printf("[CRITAL] parse log json data[%v] error: %v", jsonData, err.Error())
+		return nil
+	}
+
+	resourceType := ""
+	if v, ok := data["resourceType"]; ok {
+		resourceType = v.(string)
+	}
+	resourceName := ""
+	if v, ok := data["resourceName"]; ok {
+		resourceName = v.(string)
+		if resourceName != "" {
+			resourceName = strings.Split(resourceName, "/")[0]
+		}
+	}
+	accountId, principalId, userName := parseUserIdentityFields(data)
+
+	return &ResourceAccountInfo{
+		ResourceType: resourceType,
+		ResourceName: resourceName,
+		AccountId:    accountId,
+		PrincipalId:  principalId,
+		UserName:     userName,
+	}
+}
+
+func parseUserIdentityFields(data map[string]interface{}) (accountId, principalId, userName string) {
+	if v, ok := data["userIdentity.accountId"]; ok {
+		accountId = v.(string)
+	}
+	if v, ok := data["userIdentity.principalId"]; ok {
+		principalId = v.(string)
+	}
+	if v, ok := data["userIdentity.userName"]; ok {
+		userName = v.(string)
+	}
+	if v, ok := data["userIdentity"]; ok {
+		switch v := v.(type) {
+		case string:
+			var userIdentity map[string]string
+			err := json.Unmarshal([]byte(v), &userIdentity)
+			if err == nil {
+				accountId = userIdentity["accountId"]
+				principalId = userIdentity["principalId"]
+				userName = userIdentity["userName"]
+			}
+		}
+	}
+	return
+}

tencentcloud/common/file_common.go

@@ -13,8 +13,8 @@ const (
 	SweeperNonKeepResourceScanDir = "../../../tmp/non_keep_resource_scan/"
 )
 
-var ResourceScanHeader = []string{"资源类型", "资源名称", "实例ID", "实例名称", "分类", "创建时长(天)"}
-var NonKeepResourceScanHeader = []string{"ResourceType", "ResourceName", "InstanceId", "InstanceName"}
+var ResourceScanHeader = []string{"资源类型", "资源名称", "实例ID", "实例名称", "分类", "创建时长(天)", "创建者用户ID", "创建者用户名"}
+var NonKeepResourceScanHeader = []string{"ResourceType", "ResourceName", "InstanceId", "InstanceName", "PrincipalId", "UserName"}
 
 // WriteCsvFileData write data to csv file
 func WriteCsvFileData(dirPath string, header []string, data [][]string) error {

tencentcloud/common/resource_scan.go

@@ -6,6 +6,8 @@ import (
 	"regexp"
 	"strconv"
 	"time"
+
+	"github.com/tencentcloudstack/terraform-provider-tencentcloud/tencentcloud/connectivity"
 )
 
 const (
@@ -27,14 +29,16 @@ type ResourceInstance struct {
 	DefaultKeep bool
 }
 
-func ProcessScanCloudResources(resources, nonKeepResources []*ResourceInstance, resourceType, resourceName string) {
-	ProcessResources(resources, resourceType, resourceName)
+func ProcessScanCloudResources(client *connectivity.TencentCloudClient, resources, nonKeepResources []*ResourceInstance, resourceCreateAction string) {
+	ProcessResources(client, resources, resourceCreateAction)
 
-	ProcessNonKeepResources(nonKeepResources, resourceType, resourceName)
+	ProcessNonKeepResources(client, nonKeepResources, resourceCreateAction)
 }
 
 // ProcessResources Process all scanned cloud resources
-func ProcessResources(resources []*ResourceInstance, resourceType, resourceName string) {
+func ProcessResources(client *connectivity.TencentCloudClient, resources []*ResourceInstance, resourceCreateAction string) {
+	resourceIdToSubAccountInfoMap := GetResourceCreatorAccountInfo(client, resourceCreateAction, resources)
+
 	data := make([][]string, len(resources))
 	for i, r := range resources {
 		isResourceKeep := CheckResourceNameKeep(r.Name)
@@ -48,13 +52,24 @@ func ProcessResources(resources []*ResourceInstance, resourceType, resourceName
 			log.Printf("[CRITAL] compute resource creation duration error: %v", err.Error())
 		}
 
+		var resourceType, resourceName, principalId, userName string
+		creatorAccountInfo := resourceIdToSubAccountInfoMap[r.Id]
+		if creatorAccountInfo != nil {
+			resourceType = creatorAccountInfo.ResourceType
+			resourceName = creatorAccountInfo.ResourceName
+			principalId = creatorAccountInfo.PrincipalId
+			userName = creatorAccountInfo.UserName
+		}
+
 		data[i] = []string{
 			resourceType,
 			resourceName,
 			r.Id,
 			r.Name,
 			isResourceKeep,
 			creationDuration,
+			principalId,
+			userName,
 		}
 	}
 	err := WriteCsvFileData(SweeperResourceScanDir, ResourceScanHeader, data)
@@ -64,14 +79,27 @@ func ProcessResources(resources []*ResourceInstance, resourceType, resourceName
 }
 
 // ProcessNonKeepResources Processing scanned non-keep cloud resources
-func ProcessNonKeepResources(nonKeepResources []*ResourceInstance, resourceType, resourceName string) {
+func ProcessNonKeepResources(client *connectivity.TencentCloudClient, nonKeepResources []*ResourceInstance, resourceCreateAction string) {
+	resourceIdToSubAccountInfoMap := GetResourceCreatorAccountInfo(client, resourceCreateAction, nonKeepResources)
+
 	data := make([][]string, len(nonKeepResources))
 	for i, r := range nonKeepResources {
+		var resourceType, resourceName, principalId, userName string
+		creatorAccountInfo := resourceIdToSubAccountInfoMap[r.Id]
+		if creatorAccountInfo != nil {
+			resourceType = creatorAccountInfo.ResourceType
+			resourceName = creatorAccountInfo.ResourceName
+			principalId = creatorAccountInfo.PrincipalId
+			userName = creatorAccountInfo.UserName
+		}
+
 		data[i] = []string{
 			resourceType,
 			resourceName,
 			r.Id,
 			r.Name,
+			principalId,
+			userName,
 		}
 	}
 	err := WriteCsvFileData(SweeperNonKeepResourceScanDir, NonKeepResourceScanHeader, data)
@@ -133,7 +161,7 @@ func ParsedTime(createTime string) (*time.Time, error) {
 	} else {
 		// try parsing input strings using different time formats
 		for _, format := range TimeFormats {
-			parsedTime, err = time.Parse(format, createTime)
+			parsedTime, err = time.ParseInLocation(format, createTime, time.Local)
 			if err == nil {
 				break
 			}

tencentcloud/connectivity/client.go

@@ -8,6 +8,7 @@ import (
 	"strconv"
 	"time"
 
+	billing "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/billing/v20180709"
 	dasb "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dasb/v20191018"
 
 	oceanus "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/oceanus/v20190422"
@@ -199,6 +200,7 @@ type TencentCloudClient struct {
 	trocketConn        *trocket.Client
 	biConn             *bi.Client
 	cdwpgConn          *cdwpg.Client
+	billingConn        *billing.Client
 	//internal version: replace client begin, please do not modify this annotation and refrain from inserting any code between the beginning and end lines of the annotation.
 	//internal version: replace client end, please do not modify this annotation and refrain from inserting any code between the beginning and end lines of the annotation.
 }
@@ -1376,6 +1378,20 @@ func (me *TencentCloudClient) UseCdwpgClient() *cdwpg.Client {
 	return me.cdwpgConn
 }
 
+// UseBillingClient returns billing client for service
+func (me *TencentCloudClient) UseBillingClient() *billing.Client {
+	if me.billingConn != nil {
+		return me.billingConn
+	}
+
+	cpf := me.NewClientProfile(300)
+	cpf.Language = "zh-CN"
+	me.billingConn, _ = billing.NewClient(me.Credential, me.Region, cpf)
+	me.billingConn.WithHttpTransport(&LogRoundTripper{})
+
+	return me.billingConn
+}
+
 //internal version: replace useClient begin, please do not modify this annotation and refrain from inserting any code between the beginning and end lines of the annotation.
 //internal version: replace useClient end, please do not modify this annotation and refrain from inserting any code between the beginning and end lines of the annotation.
 

tencentcloud/services/as/resource_tc_as_attachment_test.go

@@ -28,7 +28,8 @@ func testSweepAsAttachment(r string) error {
 	logId := tccommon.GetLogId(tccommon.ContextNil)
 	ctx := context.WithValue(context.TODO(), tccommon.LogIdKey, logId)
 	cli, _ := tcacctest.SharedClientForRegion(r)
-	asService := svcas.NewAsService(cli.(tccommon.ProviderMeta).GetAPIV3Conn())
+	client := cli.(tccommon.ProviderMeta).GetAPIV3Conn()
+	asService := svcas.NewAsService(client)
 
 	scalingGroups, err := asService.DescribeAutoScalingGroupByFilter(ctx, "", "", "", nil)
 	if err != nil {
@@ -50,7 +51,7 @@ func testSweepAsAttachment(r string) error {
 			CreatTime: *v.CreatedTime,
 		})
 	}
-	tccommon.ProcessScanCloudResources(resources, nonKeepResources, "as", "attachment")
+	tccommon.ProcessScanCloudResources(client, resources, nonKeepResources, "CreateAutoScalingGroup")
 
 	for _, v := range scalingGroups {
 		scalingGroupId := *v.AutoScalingGroupId

tencentcloud/services/cam/resource_tc_cam_group_test.go

@@ -50,7 +50,7 @@ func init() {
 					CreatTime: *v.CreateTime,
 				})
 			}
-			tccommon.ProcessScanCloudResources(resources, nonKeepResources, "cam", "group")
+			tccommon.ProcessScanCloudResources(client, resources, nonKeepResources, "CreateGroup")
 
 			for _, v := range groups {
 				name := *v.GroupName

tencentcloud/services/cbs/resource_tc_cbs_storage_test.go

@@ -46,7 +46,7 @@ func init() {
 					CreatTime: *v.CreateTime,
 				})
 			}
-			tccommon.ProcessScanCloudResources(resources, nonKeepResources, "cbs", "storage")
+			tccommon.ProcessScanCloudResources(client, resources, nonKeepResources, "CreateDisks")
 
 			for i := range disks {
 				disk := disks[i]