Support optional resultset metadata

Allow optional resultset metadata.
Can potentially improve the performance in many scenario.

Issue #1105

Author: tz70s

AUTHORS

@@ -90,6 +90,7 @@ Tan Jinhua <312841925 at qq.com>
 Thomas Wodarek <wodarekwebpage at gmail.com>
 Tim Ruffles <timruffles at gmail.com>
 Tom Jenkinson <tom at tjenkinson.me>
+Tzu-Chiao Yeh <su3g4284zo6y7 at gmail.com>
 Vladimir Kovpak <cn007b at gmail.com>
 Xiangyu Hu <xiangyu.hu at outlook.com>
 Xiaobing Jiang <s7v7nislands at gmail.com>

README.md

@@ -282,6 +282,17 @@ Allow multiple statements in one query. While this allows batch queries, it also
 
 When `multiStatements` is used, `?` parameters must only be used in the first statement.
 
+##### `resultSetMetadata`
+
+```
+Type:           string
+Valid Values:   "full", "none"
+Default:        empty
+```
+
+Allow resultset metadata being optional. 
+By making resultset metadata transfer being optional, can potentially improve queries performance.
+
 ##### `parseTime`
 
 ```

connection.go

@@ -21,20 +21,21 @@ import (
 )
 
 type mysqlConn struct {
-	buf              buffer
-	netConn          net.Conn
-	rawConn          net.Conn // underlying connection when netConn is TLS connection.
-	affectedRows     uint64
-	insertId         uint64
-	cfg              *Config
-	maxAllowedPacket int
-	maxWriteSize     int
-	writeTimeout     time.Duration
-	flags            clientFlag
-	status           statusFlag
-	sequence         uint8
-	parseTime        bool
-	reset            bool // set when the Go SQL package calls ResetSession
+	buf               buffer
+	netConn           net.Conn
+	rawConn           net.Conn // underlying connection when netConn is TLS connection.
+	affectedRows      uint64
+	insertId          uint64
+	cfg               *Config
+	maxAllowedPacket  int
+	maxWriteSize      int
+	writeTimeout      time.Duration
+	flags             clientFlag
+	status            statusFlag
+	sequence          uint8
+	parseTime         bool
+	reset             bool // set when the Go SQL package calls ResetSession
+	resultSetMetadata uint8
 
 	// for context support (Go 1.8+)
 	watching bool
@@ -392,6 +393,10 @@ func (mc *mysqlConn) query(query string, args []driver.Value) (*textRows, error)
 				}
 			}
 
+			if mc.cfg.ResultSetMetadata != "" && mc.resultSetMetadata == resultSetMetadataNone {
+				return mc.readIgnoreColumns(rows, resLen)
+			}
+
 			// Columns
 			rows.rs.columns, err = mc.readColumns(resLen)
 			return rows, err
@@ -400,6 +405,21 @@ func (mc *mysqlConn) query(query string, args []driver.Value) (*textRows, error)
 	return nil, mc.markBadConn(err)
 }
 
+func (mc *mysqlConn) readIgnoreColumns(rows *textRows, resLen int) (*textRows, error) {
+	data, err := mc.readPacket()
+	if err != nil {
+		errLog.Print(err)
+		return nil, err
+	}
+	// Expected an EOF packet
+	if data[0] == iEOF && (len(data) == 5 || len(data) == 1) {
+		// Set empty columnNames, we will first read these columnNames via rows.Columns().
+		rows.rs.columnNames = make([]string, resLen)
+		return rows, nil
+	}
+	return nil, ErrOptionalResultSet
+}
+
 // Gets the value of the given MySQL System Variable
 // The returned byte slice is only valid until the next read
 func (mc *mysqlConn) getSystemVar(name string) ([]byte, error) {

connector.go

@@ -129,6 +129,24 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) {
 		mc.maxWriteSize = mc.maxAllowedPacket
 	}
 
+	// Additional handling for result set optional metadata
+	if mc.cfg.ResultSetMetadata != "" {
+		err = mc.exec("SET resultset_metadata=" + mc.cfg.ResultSetMetadata)
+		if err != nil {
+			mc.Close()
+			return nil, err
+		}
+		switch mc.cfg.ResultSetMetadata {
+		case resultSetMetadataSysVarNone:
+			mc.resultSetMetadata = resultSetMetadataNone
+		case resultSetMetadataSysVarFull:
+			mc.resultSetMetadata = resultSetMetadataFull
+		default:
+			mc.Close()
+			return nil, ErrOptionalResultSet
+		}
+	}
+
 	// Handle DSN Params
 	err = mc.handleParams()
 	if err != nil {

const.go

@@ -56,6 +56,7 @@ const (
 	clientCanHandleExpiredPasswords
 	clientSessionTrack
 	clientDeprecateEOF
+	clientOptionalResultSetMetadata
 )
 
 const (
@@ -172,3 +173,16 @@ const (
 	cachingSha2PasswordFastAuthSuccess           = 3
 	cachingSha2PasswordPerformFullAuthentication = 4
 )
+
+const (
+	// One-byte metadata flag
+	// https://dev.mysql.com/worklog/task/?id=8134
+	resultSetMetadataNone uint8 = iota
+	resultSetMetadataFull
+)
+
+const (
+	// ResultSet Metadata system var
+	resultSetMetadataSysVarNone = "NONE"
+	resultSetMetadataSysVarFull = "FULL"
+)

driver_test.go

@@ -43,6 +43,7 @@ var (
 	prot      string
 	addr      string
 	dbname    string
+	vendor    string
 	dsn       string
 	netAddr   string
 	available bool
@@ -1344,6 +1345,45 @@ func TestFoundRows(t *testing.T) {
 	})
 }
 
+func TestOptionalResultSetMetadata(t *testing.T) {
+	runTests(t, dsn+"&resultSetMetadata=none", func(dbt *DBTest) {
+		_, err := dbt.db.Exec("CREATE TABLE test (id INT NOT NULL ,data INT NOT NULL)")
+		// Error 1193: Unknown system variable 'resultset_metadata' => skip test,
+		// MySQL server version is too old
+		maybeSkip(t, err, 1193)
+		dbt.mustExec("INSERT INTO test (id, data) VALUES (0, 0),(0, 0),(1, 0),(1, 0),(1, 1)")
+
+		row := dbt.db.QueryRow("SELECT id, data FROM test WHERE id = 1")
+		id, data := 0, 0
+		err = row.Scan(&id, &data)
+		if err != nil {
+			dbt.Fatal(err)
+		}
+
+		if id != 1 && data != 0 {
+			dbt.Fatal("invalid result")
+		}
+	})
+	runTests(t, dsn+"&resultSetMetadata=full", func(dbt *DBTest) {
+		_, err := dbt.db.Exec("CREATE TABLE test (id INT NOT NULL ,data INT NOT NULL)")
+		// Error 1193: Unknown system variable 'resultset_metadata' => skip test,
+		// MySQL server version is too old
+		maybeSkip(t, err, 1193)
+		dbt.mustExec("INSERT INTO test (id, data) VALUES (0, 0),(0, 0),(1, 0),(1, 0),(1, 1)")
+
+		row := dbt.db.QueryRow("SELECT id, data FROM test WHERE id = 1")
+		id, data := 0, 0
+		err = row.Scan(&id, &data)
+		if err != nil {
+			dbt.Fatal(err)
+		}
+
+		if id != 1 && data != 0 {
+			dbt.Fatal("invalid result")
+		}
+	})
+}
+
 func TestTLS(t *testing.T) {
 	tlsTestReq := func(dbt *DBTest) {
 		if err := dbt.db.Ping(); err != nil {

dsn.go

@@ -34,22 +34,23 @@ var (
 // If a new Config is created instead of being parsed from a DSN string,
 // the NewConfig function should be used, which sets default values.
 type Config struct {
-	User             string            // Username
-	Passwd           string            // Password (requires User)
-	Net              string            // Network type
-	Addr             string            // Network address (requires Net)
-	DBName           string            // Database name
-	Params           map[string]string // Connection parameters
-	Collation        string            // Connection collation
-	Loc              *time.Location    // Location for time.Time values
-	MaxAllowedPacket int               // Max packet size allowed
-	ServerPubKey     string            // Server public key name
-	pubKey           *rsa.PublicKey    // Server public key
-	TLSConfig        string            // TLS configuration name
-	tls              *tls.Config       // TLS configuration
-	Timeout          time.Duration     // Dial timeout
-	ReadTimeout      time.Duration     // I/O read timeout
-	WriteTimeout     time.Duration     // I/O write timeout
+	User              string            // Username
+	Passwd            string            // Password (requires User)
+	Net               string            // Network type
+	Addr              string            // Network address (requires Net)
+	DBName            string            // Database name
+	Params            map[string]string // Connection parameters
+	Collation         string            // Connection collation
+	Loc               *time.Location    // Location for time.Time values
+	MaxAllowedPacket  int               // Max packet size allowed
+	ServerPubKey      string            // Server public key name
+	pubKey            *rsa.PublicKey    // Server public key
+	TLSConfig         string            // TLS configuration name
+	tls               *tls.Config       // TLS configuration
+	Timeout           time.Duration     // Dial timeout
+	ReadTimeout       time.Duration     // I/O read timeout
+	WriteTimeout      time.Duration     // I/O write timeout
+	ResultSetMetadata string            // Allow optional resultset metadata
 
 	AllowAllFiles           bool // Allow all files to be used with LOAD DATA LOCAL INFILE
 	AllowCleartextPasswords bool // Allows the cleartext client side plugin
@@ -240,6 +241,10 @@ func (cfg *Config) FormatDSN() string {
 		writeDSNParam(&buf, &hasParam, "multiStatements", "true")
 	}
 
+	if cfg.ResultSetMetadata != "" {
+		writeDSNParam(&buf, &hasParam, "resultSetMetadata", strings.ToLower(cfg.ResultSetMetadata))
+	}
+
 	if cfg.ParseTime {
 		writeDSNParam(&buf, &hasParam, "parseTime", "true")
 	}
@@ -465,6 +470,18 @@ func parseDSNParams(cfg *Config, params string) (err error) {
 				return errors.New("invalid bool value: " + value)
 			}
 
+		// allow resultset metadata being optional
+		case "resultSetMetadata":
+			// Pre-check resultSetMetadata.
+			// Although so far there's only two modes FULL and NONE, in the future it may be extended.
+			// Because if any potential extensions introduced will force us do the read path change,
+			// failed earlier when parsing DSN.
+			upperVal := strings.ToUpper(value)
+			if upperVal != resultSetMetadataSysVarFull && upperVal != resultSetMetadataSysVarNone {
+				return errors.New("invalid resultset metadata, allow FULL and NONE only")
+			}
+			cfg.ResultSetMetadata = upperVal
+
 		// time.Time parsing
 		case "parseTime":
 			var isBool bool

dsn_test.go

@@ -44,6 +44,9 @@ var testDSNs = []struct {
 }, {
 	"user:password@/dbname?allowNativePasswords=false&checkConnLiveness=false&maxAllowedPacket=0",
 	&Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: 0, AllowNativePasswords: false, CheckConnLiveness: false},
+}, {
+	"user:password@/dbname?allowNativePasswords=false&checkConnLiveness=false&maxAllowedPacket=0&resultSetMetadata=none",
+	&Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: 0, AllowNativePasswords: false, CheckConnLiveness: false, ResultSetMetadata: "NONE"},
 }, {
 	"user:p@ss(word)@tcp([de:ad:be:ef::ca:fe]:80)/dbname?loc=Local",
 	&Config{User: "user", Passwd: "p@ss(word)", Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:80", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.Local, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true},

errors.go

@@ -21,14 +21,15 @@ var (
 	ErrMalformPkt        = errors.New("malformed packet")
 	ErrNoTLS             = errors.New("TLS requested but server does not support TLS")
 	ErrCleartextPassword = errors.New("this user requires clear text authentication. If you still want to use it, please add 'allowCleartextPasswords=1' to your DSN")
-	ErrNativePassword    = errors.New("this user requires mysql native password authentication.")
+	ErrNativePassword    = errors.New("this user requires mysql native password authentication")
 	ErrOldPassword       = errors.New("this user requires old password authentication. If you still want to use it, please add 'allowOldPasswords=1' to your DSN. See also https://github.com/go-sql-driver/mysql/wiki/old_passwords")
 	ErrUnknownPlugin     = errors.New("this authentication plugin is not supported")
 	ErrOldProtocol       = errors.New("MySQL server does not support required protocol 41+")
 	ErrPktSync           = errors.New("commands out of sync. You can't run this command now")
 	ErrPktSyncMul        = errors.New("commands out of sync. Did you run multiple statements at once?")
 	ErrPktTooLarge       = errors.New("packet for query is too large. Try adjusting the 'max_allowed_packet' variable on the server")
 	ErrBusyBuffer        = errors.New("busy buffer")
+	ErrOptionalResultSet = errors.New("malformed optional resultset metadata packets")
 
 	// errBadConnNoWrite is used for connection errors where nothing was sent to the database yet.
 	// If this happens first in a function starting a database interaction, it should be replaced by driver.ErrBadConn

packets.go

@@ -301,6 +301,10 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, plugin string
 		clientFlags |= clientMultiStatements
 	}
 
+	if mc.cfg.ResultSetMetadata != "" {
+		clientFlags |= clientOptionalResultSetMetadata
+	}
+
 	// encode length of the auth plugin data
 	var authRespLEIBuf [9]byte
 	authRespLen := len(authResp)
@@ -555,6 +559,17 @@ func (mc *mysqlConn) readResultSetHeaderPacket() (int, error) {
 			return int(num), nil
 		}
 
+		// Sniff one extra byte for resultset metadata if we set capability
+		// CLIENT_OPTIONAL_RESULTSET_METADTA
+		// https://dev.mysql.com/worklog/task/?id=8134
+		if len(data) == 2 {
+			// ResultSet metadata flag check
+			if mc.resultSetMetadata != data[1] {
+				return 0, ErrOptionalResultSet
+			}
+			return int(num), nil
+		}
+
 		return 0, ErrMalformPkt
 	}
 	return 0, err