Merge branch 'master' into reuse-buf

Author: lance6716

README.md

@@ -237,8 +237,8 @@ err := conn.ExecuteSelectStreaming(`select id, name from table LIMIT 100500`, &r
     	// Copy it if you need.
     	// ...
     }
-    return false, nil
-})
+    return nil
+}, nil)
 
 // ...
 ```

canal/canal.go

@@ -386,7 +386,7 @@ func (c *Canal) CheckBinlogRowImage(image string) error {
 	// need to check MySQL binlog row image? full, minimal or noblob?
 	// now only log
 	if c.cfg.Flavor == mysql.MySQLFlavor {
-		if res, err := c.Execute(`SHOW GLOBAL VARIABLES LIKE "binlog_row_image"`); err != nil {
+		if res, err := c.Execute(`SHOW GLOBAL VARIABLES LIKE 'binlog_row_image'`); err != nil {
 			return errors.Trace(err)
 		} else {
 			// MySQL has binlog row image from 5.6, so older will return empty
@@ -401,7 +401,7 @@ func (c *Canal) CheckBinlogRowImage(image string) error {
 }
 
 func (c *Canal) checkBinlogRowFormat() error {
-	res, err := c.Execute(`SHOW GLOBAL VARIABLES LIKE "binlog_format";`)
+	res, err := c.Execute(`SHOW GLOBAL VARIABLES LIKE 'binlog_format';`)
 	if err != nil {
 		return errors.Trace(err)
 	} else if f, _ := res.GetString(0, 1); f != "ROW" {

canal/sync.go

@@ -223,7 +223,7 @@ func parseStmt(stmt ast.StmtNode) (ns []*node) {
 	case *ast.TruncateTableStmt:
 		n := &node{
 			db:    t.Table.Schema.String(),
-			table: t.Table.Schema.String(),
+			table: t.Table.Name.String(),
 		}
 		ns = []*node{n}
 	}

client/auth.go

@@ -140,9 +140,18 @@ func (c *Conn) writeAuthHandshake() error {
 	if !authPluginAllowed(c.authPluginName) {
 		return fmt.Errorf("unknow auth plugin name '%s'", c.authPluginName)
 	}
-	// Adjust client capability flags based on server support
+
+	// Set default client capabilities that reflect the abilities of this library
 	capability := CLIENT_PROTOCOL_41 | CLIENT_SECURE_CONNECTION |
-		CLIENT_LONG_PASSWORD | CLIENT_TRANSACTIONS | CLIENT_PLUGIN_AUTH | c.capability&CLIENT_LONG_FLAG
+		CLIENT_LONG_PASSWORD | CLIENT_TRANSACTIONS | CLIENT_PLUGIN_AUTH
+	// Adjust client capability flags based on server support
+	capability |= c.capability & CLIENT_LONG_FLAG
+	// Adjust client capability flags on specific client requests
+	// Only flags that would make any sense setting and aren't handled elsewhere
+	// in the library are supported here
+	capability |= c.ccaps&CLIENT_FOUND_ROWS | c.ccaps&CLIENT_IGNORE_SPACE |
+		c.ccaps&CLIENT_MULTI_STATEMENTS | c.ccaps&CLIENT_MULTI_RESULTS |
+		c.ccaps&CLIENT_PS_MULTI_RESULTS | c.ccaps&CLIENT_CONNECT_ATTRS
 
 	// To enable TLS / SSL
 	if c.tlsConfig != nil {

client/client_test.go

@@ -1,6 +1,7 @@
 package client
 
 import (
+	"encoding/json"
 	"flag"
 	"fmt"
 	"strings"
@@ -82,6 +83,7 @@ func (s *clientTestSuite) testConn_CreateTable(c *C) {
           e enum("test1", "test2"),
           u tinyint unsigned,
           i tinyint,
+          j json,
           PRIMARY KEY (id)
         ) ENGINE=InnoDB DEFAULT CHARSET=utf8`
 
@@ -94,6 +96,41 @@ func (s *clientTestSuite) TestConn_Ping(c *C) {
 	c.Assert(err, IsNil)
 }
 
+func (s *clientTestSuite) TestConn_SetCapability(c *C) {
+	caps := []uint32{
+		mysql.CLIENT_LONG_PASSWORD,
+		mysql.CLIENT_FOUND_ROWS,
+		mysql.CLIENT_LONG_FLAG,
+		mysql.CLIENT_CONNECT_WITH_DB,
+		mysql.CLIENT_NO_SCHEMA,
+		mysql.CLIENT_COMPRESS,
+		mysql.CLIENT_ODBC,
+		mysql.CLIENT_LOCAL_FILES,
+		mysql.CLIENT_IGNORE_SPACE,
+		mysql.CLIENT_PROTOCOL_41,
+		mysql.CLIENT_INTERACTIVE,
+		mysql.CLIENT_SSL,
+		mysql.CLIENT_IGNORE_SIGPIPE,
+		mysql.CLIENT_TRANSACTIONS,
+		mysql.CLIENT_RESERVED,
+		mysql.CLIENT_SECURE_CONNECTION,
+		mysql.CLIENT_MULTI_STATEMENTS,
+		mysql.CLIENT_MULTI_RESULTS,
+		mysql.CLIENT_PS_MULTI_RESULTS,
+		mysql.CLIENT_PLUGIN_AUTH,
+		mysql.CLIENT_CONNECT_ATTRS,
+		mysql.CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA,
+	}
+
+	for _, cap := range caps {
+		c.Assert(s.c.ccaps&cap > 0, IsFalse)
+		s.c.SetCapability(cap)
+		c.Assert(s.c.ccaps&cap > 0, IsTrue)
+		s.c.UnsetCapability(cap)
+		c.Assert(s.c.ccaps&cap > 0, IsFalse)
+	}
+}
+
 // NOTE for MySQL 5.5 and 5.6, server side has to config SSL to pass the TLS test, otherwise, it will throw error that
 //      MySQL server does not support TLS required by the client. However, for MySQL 5.7 and above, auto generated certificates
 //      are used by default so that manual config is no longer necessary.
@@ -149,6 +186,14 @@ func (s *clientTestSuite) TestConn_Insert(c *C) {
 	c.Assert(pkg.AffectedRows, Equals, uint64(1))
 }
 
+func (s *clientTestSuite) TestConn_Insert2(c *C) {
+	str := `insert into mixer_test_conn (id, j) values(?, ?)`
+	j := json.RawMessage(`[]`)
+	pkg, err := s.c.Execute(str, []interface{}{2, j}...)
+	c.Assert(err, IsNil)
+	c.Assert(pkg.AffectedRows, Equals, uint64(1))
+}
+
 func (s *clientTestSuite) TestConn_Select(c *C) {
 	str := `select str, f, e from mixer_test_conn where id = 1`
 

client/conn.go

@@ -21,7 +21,10 @@ type Conn struct {
 	tlsConfig *tls.Config
 	proto     string
 
+	// server capabilities
 	capability uint32
+	// client-set capabilities only
+	ccaps uint32
 
 	status uint16
 
@@ -36,6 +39,9 @@ type Conn struct {
 // This function will be called for every row in resultset from ExecuteSelectStreaming.
 type SelectPerRowCallback func(row []FieldValue) error
 
+// This function will be called once per result from ExecuteSelectStreaming
+type SelectPerResultCallback func(result *Result) error
+
 func getNetProto(addr string) string {
 	proto := "tcp"
 	if strings.Contains(addr, "/") {
@@ -120,6 +126,16 @@ func (c *Conn) Ping() error {
 	return nil
 }
 
+// SetCapability enables the use of a specific capability
+func (c *Conn) SetCapability(cap uint32) {
+	c.ccaps |= cap
+}
+
+// UnsetCapability disables the use of a specific capability
+func (c *Conn) UnsetCapability(cap uint32) {
+	c.ccaps &= ^cap
+}
+
 // UseSSL: use default SSL
 // pass to options when connect
 func (c *Conn) UseSSL(insecureSkipVerify bool) {
@@ -170,6 +186,7 @@ func (c *Conn) Execute(command string, args ...interface{}) (*Result, error) {
 
 // ExecuteSelectStreaming will call perRowCallback for every row in resultset
 //   WITHOUT saving any row data to Result.{Values/RawPkg/RowDatas} fields.
+// When given, perResultCallback will be called once per result
 //
 // ExecuteSelectStreaming should be used only for SELECT queries with a large response resultset for memory preserving.
 //
@@ -180,14 +197,14 @@ func (c *Conn) Execute(command string, args ...interface{}) (*Result, error) {
 //   		// Use the row as you want.
 //   		// You must not save FieldValue.AsString() value after this callback is done. Copy it if you need.
 //   		return nil
-// 		})
+// 		}, nil)
 //
-func (c *Conn) ExecuteSelectStreaming(command string, result *Result, perRowCallback SelectPerRowCallback) error {
+func (c *Conn) ExecuteSelectStreaming(command string, result *Result, perRowCallback SelectPerRowCallback, perResultCallback SelectPerResultCallback) error {
 	if err := c.writeCommandStr(COM_QUERY, command); err != nil {
 		return errors.Trace(err)
 	}
 
-	return c.readResultStreaming(false, result, perRowCallback)
+	return c.readResultStreaming(false, result, perRowCallback, perResultCallback)
 }
 
 func (c *Conn) Begin() error {

client/pool.go

@@ -466,11 +466,12 @@ func (pool *Pool) startNewConnections(count int) {
 
 func (pool *Pool) ping(conn *Conn) error {
 	deadline := time.Now().Add(100 * time.Millisecond)
-	_ = conn.SetWriteDeadline(deadline)
-	_ = conn.SetReadDeadline(deadline)
+	_ = conn.SetDeadline(deadline)
 	err := conn.Ping()
 	if err != nil {
 		pool.logFunc(`Pool: ping query fail: %s`, err.Error())
+	} else {
+		_ = conn.SetDeadline(time.Time{})
 	}
 	return err
 }

client/resp.go

@@ -129,6 +129,8 @@ func (c *Conn) handleAuthResult() error {
 		if data[0] == CACHE_SHA2_FAST_AUTH {
 			if _, err = c.readOK(); err == nil {
 				return nil // auth successful
+			} else {
+				return err
 			}
 		} else if data[0] == CACHE_SHA2_FULL_AUTH {
 			// need full authentication
@@ -233,7 +235,7 @@ func (c *Conn) readResult(binary bool) (*Result, error) {
 	return c.readResultset(firstPkgBuf, binary)
 }
 
-func (c *Conn) readResultStreaming(binary bool, result *Result, perRowCb SelectPerRowCallback) error {
+func (c *Conn) readResultStreaming(binary bool, result *Result, perRowCb SelectPerRowCallback, perResCb SelectPerResultCallback) error {
 	firstPkgBuf, err := c.ReadPacketReuseMem(utils.ByteSliceGet(16)[:0])
 	defer utils.ByteSlicePut(firstPkgBuf)
 
@@ -267,7 +269,7 @@ func (c *Conn) readResultStreaming(binary bool, result *Result, perRowCb SelectP
 		return ErrMalformPacket
 	}
 
-	return c.readResultsetStreaming(firstPkgBuf, binary, result, perRowCb)
+	return c.readResultsetStreaming(firstPkgBuf, binary, result, perRowCb, perResCb)
 }
 
 func (c *Conn) readResultset(data []byte, binary bool) (*Result, error) {
@@ -293,7 +295,7 @@ func (c *Conn) readResultset(data []byte, binary bool) (*Result, error) {
 	return result, nil
 }
 
-func (c *Conn) readResultsetStreaming(data []byte, binary bool, result *Result, perRowCb SelectPerRowCallback) error {
+func (c *Conn) readResultsetStreaming(data []byte, binary bool, result *Result, perRowCb SelectPerRowCallback, perResCb SelectPerResultCallback) error {
 	columnCount, _, n := LengthEncodedInt(data)
 
 	if n-len(data) != 0 {
@@ -307,14 +309,26 @@ func (c *Conn) readResultsetStreaming(data []byte, binary bool, result *Result,
 		result.Reset(int(columnCount))
 	}
 
+	// this is a streaming resultset
+	result.Resultset.Streaming = true
+
 	if err := c.readResultColumns(result); err != nil {
 		return errors.Trace(err)
 	}
 
+	if perResCb != nil {
+		if err := perResCb(result); err != nil {
+			return err
+		}
+	}
+
 	if err := c.readResultRowsStreaming(result, binary, perRowCb); err != nil {
 		return errors.Trace(err)
 	}
 
+	// this resultset is done streaming
+	result.Resultset.StreamingDone = true
+
 	return nil
 }
 

client/stmt.go

@@ -2,6 +2,7 @@ package client
 
 import (
 	"encoding/binary"
+	"encoding/json"
 	"fmt"
 	"math"
 
@@ -38,6 +39,14 @@ func (s *Stmt) Execute(args ...interface{}) (*Result, error) {
 	return s.conn.readResult(true)
 }
 
+func (s *Stmt) ExecuteSelectStreaming(result *Result, perRowCb SelectPerRowCallback, perResCb SelectPerResultCallback, args ...interface{}) error {
+	if err := s.write(args...); err != nil {
+		return errors.Trace(err)
+	}
+
+	return s.conn.readResultStreaming(true, result, perRowCb, perResCb)
+}
+
 func (s *Stmt) Close() error {
 	if err := s.conn.writeCommandUint32(COM_STMT_CLOSE, s.id); err != nil {
 		return errors.Trace(err)
@@ -127,6 +136,9 @@ func (s *Stmt) write(args ...interface{}) error {
 		case []byte:
 			paramTypes[i<<1] = MYSQL_TYPE_STRING
 			paramValues[i] = append(PutLengthEncodedInt(uint64(len(v))), v...)
+		case json.RawMessage:
+			paramTypes[i<<1] = MYSQL_TYPE_STRING
+			paramValues[i] = append(PutLengthEncodedInt(uint64(len(v))), v...)
 		default:
 			return fmt.Errorf("invalid argument type %T", args[i])
 		}

cmd/go-mysqlbinlog/main.go

@@ -24,6 +24,7 @@ var flavor = flag.String("flavor", "mysql", "Flavor: mysql or mariadb")
 
 var file = flag.String("file", "", "Binlog filename")
 var pos = flag.Int("pos", 4, "Binlog position")
+var gtid = flag.String("gtid", "", "Binlog GTID set that this slave has executed")
 
 var semiSync = flag.Bool("semisync", false, "Support semi sync")
 var backupPath = flag.String("backup_path", "", "backup path to store binlog files")
@@ -57,10 +58,27 @@ func main() {
 			return
 		}
 	} else {
-		s, err := b.StartSync(pos)
-		if err != nil {
-			fmt.Printf("Start sync error: %v\n", errors.ErrorStack(err))
-			return
+		var (
+			s   *replication.BinlogStreamer
+			err error
+		)
+		if len(*gtid) > 0 {
+			gset, err := mysql.ParseGTIDSet(*flavor, *gtid)
+			if err != nil {
+				fmt.Printf("Failed to parse gtid %s with flavor %s, error: %v\n",
+					*gtid, *flavor, errors.ErrorStack(err))
+			}
+			s, err = b.StartSyncGTID(gset)
+			if err != nil {
+				fmt.Printf("Start sync by GTID error: %v\n", errors.ErrorStack(err))
+				return
+			}
+		} else {
+			s, err = b.StartSync(pos)
+			if err != nil {
+				fmt.Printf("Start sync error: %v\n", errors.ErrorStack(err))
+				return
+			}
 		}
 
 		for {

dump/dumper.go

@@ -164,7 +164,9 @@ func (d *Dumper) Dump(w io.Writer) error {
 	}
 
 	args = append(args, fmt.Sprintf("--user=%s", d.User))
-	args = append(args, fmt.Sprintf("--password=%s", d.Password))
+	passwordArg := fmt.Sprintf("--password=%s", d.Password)
+	args = append(args, passwordArg)
+	passwordArgIndex := len(args) - 1
 
 	if !d.masterDataSkipped {
 		args = append(args, "--master-data")
@@ -238,7 +240,9 @@ func (d *Dumper) Dump(w io.Writer) error {
 		}
 	}
 
+	args[passwordArgIndex] = "--password=******"
 	log.Infof("exec mysqldump with %v", args)
+	args[passwordArgIndex] = passwordArg
 	cmd := exec.Command(d.ExecutionPath, args...)
 
 	cmd.Stderr = d.ErrOut

mysql/mysql_test.go

@@ -223,3 +223,9 @@ func (t *mysqlTestSuite) TestMysqlUUIDClone(c *check.C) {
 	clone := us.Clone()
 	c.Assert(clone.String(), check.Equals, "de278ad0-2106-11e4-9f8e-6edd0ca20947:1-2")
 }
+
+func (t *mysqlTestSuite) TestMysqlEmptyDecode(c *check.C) {
+	_, isNull, n := LengthEncodedInt(nil)
+	c.Assert(isNull, check.IsTrue)
+	c.Assert(n, check.Equals, 0)
+}

mysql/resultset.go

@@ -17,6 +17,9 @@ type Resultset struct {
 	RawPkg []byte
 
 	RowDatas []RowData
+
+	Streaming     bool
+	StreamingDone bool
 }
 
 var (