Merge branch 'master' into PMM-2569_MySQL_8

Author: arvenil

.travis.yml

@@ -40,13 +40,6 @@ env:
     - MONGODB_IMAGE=percona/percona-server-mongodb:3.4
     - MONGODB_IMAGE=percona/percona-server-mongodb:3.6
 
-matrix:
-  include:
-    - go: master
-      env:
-  allow_failures:
-    - go: master
-
 addons:
   apt:
     sources:
@@ -67,7 +60,7 @@ before_script:
   # run docker containers
   - docker-compose up -d
   # wait for MySQL to become available
-  - ./with_backoff.sh docker-compose exec mysql mysql -u"${MYSQL_USER}" -e 'SELECT 1'
+  - ./with_backoff.sh docker-compose exec mysql mysql --protocol=tcp -u"${MYSQL_USER}" -e 'SELECT 1'
   # log versions
   - docker --version
   - docker-compose --version

agent/agent_test.go

@@ -287,7 +287,6 @@ func (s *AgentTestSuite) TestStartStopService(t *C) {
 		Interval:       60,         // seconds
 		MaxSlowLogSize: 1073741824, // 1 GiB
 		ExampleQueries: &exampleQueries,
-		WorkerRunTime:  120, // seconds
 	}
 
 	// Second, the service config is encoded and encapsulated in a ServiceData:
@@ -385,7 +384,6 @@ func (s *AgentTestSuite) TestStartServiceSlow(t *C) {
 		Interval:       60,         // seconds
 		MaxSlowLogSize: 1073741824, // 1 GiB
 		ExampleQueries: &exampleQueries,
-		WorkerRunTime:  120, // seconds
 	}
 	qanConfigData, _ := json.Marshal(qanConfig)
 	serviceCmd := &proto.ServiceData{

docs/agent.rst

@@ -141,8 +141,6 @@ Stop                (varies)    List of MySQL queries to execute to un-configure
 
 Interval            60          How often to collect and aggregate data
 
-WorkerRunTime       55          Max runtime for each worker per interval
-
 MaxSlowLogSize      1073741824  Rotate slow log when it becomes this large (bytes)
 
 RemoveOldSlowLogs   true        Remove slow log after rotating if true

glide.lock

@@ -21,6 +21,7 @@ import:
   - src/go/mongolib/stats
 - package: github.com/percona/pmgo
 - package: github.com/percona/pmm
+  version: 9cb15efc1ab783e17a03174254356a7b800cd1bd
   subpackages:
   - proto
   - proto/config

glide.yaml

@@ -129,7 +129,6 @@ func (s *AnalyzerTestSuite) SetUpTest(t *C) {
 		UUID:            s.mysqlUUID,
 		CollectFrom:     "slowlog",
 		Interval:        60,
-		WorkerRunTime:   60,
 		MaxSlowLogSize:  MAX_SLOW_LOG_SIZE,
 		SlowLogRotation: &slowLogRotation,
 		Start: []string{

qan/analyzer/mysql/analyzer_test.go

@@ -19,6 +19,7 @@ package config
 
 import (
 	"database/sql/driver"
+	"encoding/json"
 	"fmt"
 	"strings"
 
@@ -27,15 +28,7 @@ import (
 )
 
 var (
-	DefaultInterval          uint  = 60         // 1 minute
-	DefaultMaxSlowLogSize    int64 = 1073741824 // 1G
-	DefaultSlowLogRotation         = true       // whether to rotate slow logs
-	DefaultRemoveOldSlowLogs       = true       // whether to remove old slow logs after rotation
-	DefaultRetainSlowLogs          = 1          // how many slow logs to keep on filesystem
-	DefaultExampleQueries          = true
-	// internal
-	DefaultWorkerRuntime uint = 55
-	DefaultReportLimit   uint = 200
+	DefaultRemoveOldSlowLogs = true // whether to remove old slow logs after rotation
 )
 
 type MySQLVarType int
@@ -109,34 +102,23 @@ func ReadInfoFromShowGlobalStatus(conn mysql.Connector) (info map[string]interfa
 }
 
 func ValidateConfig(setConfig pc.QAN) (pc.QAN, error) {
-	runConfig := pc.QAN{
-		UUID:           setConfig.UUID,
-		Interval:       DefaultInterval,
-		ExampleQueries: new(bool),
-		// "slowlog" specific options.
-		MaxSlowLogSize:  DefaultMaxSlowLogSize,
-		SlowLogRotation: new(bool),
-		RetainSlowLogs:  new(int),
-		// internal
-		WorkerRunTime: DefaultWorkerRuntime,
-		ReportLimit:   DefaultReportLimit,
+	runConfig := pc.NewQAN()
+	fmt.Printf("%+v\n", runConfig)
+
+	// Marshal setConfig and unmarshal it back on default config.
+	// This way we keep defaults if they are not set in setConfig.
+	b, err := json.Marshal(setConfig)
+	if err != nil {
+		return runConfig, err
 	}
-	// I know this is an ugly hack, but we need runConfig.ExampleQueries to be a pointer since
-	// the default value for a boolean is false, there is no way to tell if it was false in the
-	// config or if the value was missing.
-	// If it was missing (nil) we should take the default=true
-	*runConfig.ExampleQueries = DefaultExampleQueries
-	if setConfig.ExampleQueries != nil {
-		runConfig.ExampleQueries = setConfig.ExampleQueries
-	}
-	*runConfig.SlowLogRotation = DefaultSlowLogRotation
-	if setConfig.SlowLogRotation != nil {
-		runConfig.SlowLogRotation = setConfig.SlowLogRotation
-	}
-	*runConfig.RetainSlowLogs = DefaultRetainSlowLogs
-	if setConfig.RetainSlowLogs != nil {
-		runConfig.RetainSlowLogs = setConfig.RetainSlowLogs
+	err = json.Unmarshal(b, &runConfig)
+	if err != nil {
+		return runConfig, err
 	}
+	fmt.Printf("%+v\n", runConfig)
+
+	// Set UUID.
+	runConfig.UUID = setConfig.UUID
 
 	// Strings
 	if setConfig.CollectFrom != "slowlog" && setConfig.CollectFrom != "perfschema" {
@@ -152,8 +134,6 @@ func ValidateConfig(setConfig pc.QAN) (pc.QAN, error) {
 		runConfig.Interval = setConfig.Interval
 	}
 
-	runConfig.WorkerRunTime = uint(float64(runConfig.Interval) * 0.9) // 90% of interval
-
 	return runConfig, nil
 }
 

qan/analyzer/mysql/config/config.go

@@ -15,7 +15,6 @@ func TestValidateConfig(t *testing.T) {
 		Interval:       300,        // 5 min
 		MaxSlowLogSize: 1073741824, // 1 GiB
 		ExampleQueries: &exampleQueries,
-		WorkerRunTime:  600, // 10 min
 		CollectFrom:    "slowlog",
 	}
 	_, err := ValidateConfig(cfg)

qan/analyzer/mysql/config/config_test.go

@@ -190,12 +190,11 @@ func (m *MySQLAnalyzer) GetDefaults(uuid string) map[string]interface{} {
 	cfg := map[string]interface{}{
 		"CollectFrom":     m.config.CollectFrom,
 		"Interval":        m.config.Interval,
-		"MaxSlowLogSize":  config.DefaultMaxSlowLogSize,
+		"MaxSlowLogSize":  m.config.MaxSlowLogSize,
 		"RetainSlowLogs":  m.config.RetainSlowLogs,
 		"SlowLogRotation": m.config.SlowLogRotation,
 		"ExampleQueries":  m.config.ExampleQueries,
-		"WorkerRunTime":   config.DefaultWorkerRuntime,
-		"ReportLimit":     config.DefaultReportLimit,
+		"ReportLimit":     m.config.ReportLimit,
 	}
 
 	// Info from SHOW GLOBAL STATUS

qan/analyzer/mysql/mysql.go

@@ -96,7 +96,6 @@ func (s *WorkerTestSuite) SetUpSuite(t *C) {
 		Interval:       60,         // 1 min
 		MaxSlowLogSize: 1073741824, // 1 GiB
 		ExampleQueries: &exampleQueries,
-		WorkerRunTime:  60, // 1 min
 		CollectFrom:    "slowlog",
 	}
 	s.nullmysql = mock.NewNullMySQL()
@@ -294,7 +293,6 @@ func (s *WorkerTestSuite) TestRotateAndRemoveSlowLog(t *C) {
 		ExampleQueries:  &exampleQueries,
 		SlowLogRotation: &slowLogsRotation,
 		RetainSlowLogs:  &slowLogsToKeep,
-		WorkerRunTime:   600,
 		Start: []string{
 			"-- start",
 		},
@@ -397,7 +395,6 @@ func (s *WorkerTestSuite) TestRotateSlowLog(t *C) {
 		ExampleQueries:  &exampleQueries,
 		SlowLogRotation: &slowLogsRotation,
 		RetainSlowLogs:  &slowLogsToKeep,
-		WorkerRunTime:   600,
 		Start: []string{
 			"-- start",
 		},
@@ -552,7 +549,6 @@ func (s *WorkerTestSuite) TestRotateRealSlowLog(t *C) {
 		Interval:       300,
 		MaxSlowLogSize: 1000,
 		ExampleQueries: &exampleQueries,
-		WorkerRunTime:  600,
 		Start: []string{
 			"SET GLOBAL slow_query_log=1",
 			fmt.Sprintf("SET GLOBAL slow_query_log_file='%s'", slowlogFile),
@@ -629,7 +625,6 @@ func (s *WorkerTestSuite) TestStop(t *C) {
 		UUID:           s.mysqlInstance.UUID,
 		Interval:       300,
 		MaxSlowLogSize: 1024 * 1024 * 1024,
-		WorkerRunTime:  60,
 		Start:          []string{},
 		Stop:           []string{},
 		CollectFrom:    "slowlog",
@@ -716,7 +711,6 @@ func (s *WorkerTestSuite) TestResult014(t *C) {
 		UUID:           "1",
 		CollectFrom:    "slowlog",
 		Interval:       60,
-		WorkerRunTime:  60,
 		ReportLimit:    500,
 		MaxSlowLogSize: 1024 * 1024 * 1000,
 	}

qan/analyzer/mysql/worker/slowlog/slowlog_test.go

@@ -155,13 +155,14 @@ func (w *Worker) Setup(interval *iter.Interval) error {
 		}
 	}
 
+	workerRunTime := time.Duration(uint(float64(w.config.Interval)*0.9)) * time.Second // 90% of interval
 	// Create new Job.
 	w.job = &Job{
 		Id:             fmt.Sprintf("%d", interval.Number),
 		SlowLogFile:    interval.Filename,
 		StartOffset:    interval.StartOffset,
 		EndOffset:      interval.EndOffset,
-		RunTime:        time.Duration(w.config.WorkerRunTime) * time.Second,
+		RunTime:        workerRunTime,
 		ExampleQueries: boolValue(w.config.ExampleQueries),
 		RetainSlowLogs: intValue(w.config.RetainSlowLogs),
 	}

qan/analyzer/mysql/worker/slowlog/worker.go

@@ -175,7 +175,6 @@ func (s *ManagerTestSuite) TestStartWithConfig(t *C) {
 			UUID:           mysqlInstance.UUID,
 			CollectFrom:    analyzerType,
 			Interval:       300,
-			WorkerRunTime:  270,
 			ExampleQueries: &exampleQueries, // specify optional args
 			ReportLimit:    200,             // specify optional args
 		}
@@ -267,7 +266,6 @@ func (s *ManagerTestSuite) TestStart2RemoteQAN(t *C) {
 			UUID:           mysqlInstance.UUID,
 			CollectFrom:    "perfschema",
 			Interval:       300,
-			WorkerRunTime:  270,
 			ExampleQueries: &exampleQueries, // specify optional args
 			ReportLimit:    200,             // specify optional args
 		}
@@ -348,10 +346,9 @@ func (s *ManagerTestSuite) TestGetConfig(t *C) {
 
 	// Write a realistic qan.conf config to disk.
 	pcQANSetExpected := pc.QAN{
-		UUID:          mysqlUUID,
-		CollectFrom:   "slowlog",
-		Interval:      300,
-		WorkerRunTime: 270,
+		UUID:        mysqlUUID,
+		CollectFrom: "slowlog",
+		Interval:    300,
 	}
 	err := pct.Basedir.WriteConfig("qan-"+mysqlUUID, &pcQANSetExpected)
 	t.Assert(err, IsNil)
@@ -441,7 +438,6 @@ func (s *ManagerTestSuite) TestAddInstance(t *C) {
 		Interval:       300,        // 5 min
 		MaxSlowLogSize: 1073741824, // 1 GiB
 		ExampleQueries: &exampleQueries,
-		WorkerRunTime:  600, // 10 min
 		CollectFrom:    "slowlog",
 	}
 
@@ -537,7 +533,6 @@ func (s *ManagerTestSuite) TestStartTool(t *C) {
 		Interval:       300,        // 5 min
 		MaxSlowLogSize: 1073741824, // 1 GiB
 		ExampleQueries: &exampleQueries,
-		WorkerRunTime:  600, // 10 min
 		CollectFrom:    "slowlog",
 	}
 

qan/manager_test.go

@@ -82,10 +82,9 @@ func TestWithRealMySQL(t *testing.T) {
 
 	// Write a realistic qan.conf config to disk.
 	pcQANSetExpected := pc.QAN{
-		UUID:          protoInstance.UUID,
-		CollectFrom:   "slowlog",
-		Interval:      300,
-		WorkerRunTime: 270,
+		UUID:        protoInstance.UUID,
+		CollectFrom: "slowlog",
+		Interval:    300,
 	}
 	err = pct.Basedir.WriteConfig("qan-"+protoInstance.UUID, &pcQANSetExpected)
 	require.NoError(t, err)