*sql.DB invalid memory address or nil pointer dereference

[mejinke]

//The file /Users/jinke/golang/src/cds_spider/common/dbpool/mysql.go

package dbpool

import (
    "database/sql"
    _ "github.com/go-sql-driver/mysql"
)

var DB *sql.DB

func init() {
    db, err := sql.Open("mysql", "root:123456@tcp(192.168.1.20:3306)/?charset=utf8")
    if err != nil {
        panic("dbpool init >> " + err.Error())
    }
    DB = db
    DB.SetMaxIdleConns(5)
}

//The file /Users/jinke/golang/src/cds_spider/newCar/koubei/koubei.go

package koubei

import (
    "cds_spider/common/dbpool"
)

type KouBei struct {
    Auto_koubei_id                                               int
    Source                                                       string
    Auto_brand_id, Auto_company_id, Auto_serial_id, Auto_type_id int
    Username                                                     string
    Price                                                        float64
    Province_id, City_id                                         int
    Buy_date, Content, Spider_url, Created                       string
    Level                                                        int
}

func (k *KouBei) Save() (insertID int64, err error) {
    stmt, err := dbpool.DB.Prepare("INSERT mains.koubei SET source=?, auto_brand_id=?, auto_company_id=?, auto_serial_id=?, auto_type_id=?, username=?, price=?, province_id=?, city_id=?, buy_date=?, content=?, level=?, spider_url=?")
    if err != nil {
        return 0, err
    }
    defer stmt.Close()

    res, err := stmt.Exec(k.Source, k.Auto_brand_id, k.Auto_company_id, k.Auto_serial_id, k.Auto_type_id, k.Username, k.Price, k.Province_id, k.City_id, k.Buy_date, k.Content, k.Level, k.Spider_url)
    if err != nil {
        return 0, err
    }
    return res.LastInsertId()
}

"stmt, err := dbpool.DB.Prepare" is Error invalid memory address or nil pointer dereference

panic: runtime error: invalid memory address or nil pointer dereference
[signal 0xb code=0x1 addr=0x41 pc=0x52fb5c]

goroutine 2208 [running]:
github.com/go-sql-driver/mysql.(*mysqlConn).writeCommandPacketUint32(0x0, 0x1048eca19, 0xc2046b8330, 0x2ba32ae35a88)
    /Users/jinke/golang/src/github.com/go-sql-driver/mysql/packets.go:327 +0x1c
github.com/go-sql-driver/mysql.(*mysqlStmt).Close(0xc200e3ba50, 0x0, 0x0)
    /Users/jinke/golang/src/github.com/go-sql-driver/mysql/statement.go:24 +0x46
database/sql.(*driverConn).finalClose(0xc2020d5cc0, 0xc20370d900, 0x2bbc7f0)
    /usr/local/go/src/pkg/database/sql/sql.go:285 +0x87
database/sql.func·002(0xc2000ca200, 0xc2000aebd0)
    /usr/local/go/src/pkg/database/sql/sql.go:372 +0x2c
database/sql.(*driverConn).Close(0xc2020d5cc0, 0x3, 0x4a173d)
    /usr/local/go/src/pkg/database/sql/sql.go:278 +0x174
database/sql.(*DB).putConn(0xc2000ca1e0, 0xc2020d5cc0, 0x0, 0x0)
    /usr/local/go/src/pkg/database/sql/sql.go:598 +0x2e2
database/sql.(*DB).prepare(0xc2000ca1e0, 0x742450, 0xd2, 0xc20035ba50, 0x85, ...)
    /usr/local/go/src/pkg/database/sql/sql.go:640 +0x267
database/sql.(*DB).Prepare(0xc2000ca1e0, 0x742450, 0xd2, 0x4b2488, 0xc2000ae140, ...)
    /usr/local/go/src/pkg/database/sql/sql.go:608 +0x5a
cds_spider/newCar/koubei.(*KouBei).Save(0xc2046ab790, 0x0, 0x0, 0x0)
    /Users/jinke/golang/src/cds_spider/newCar/koubei/koubei.go:20 +0x6b

[julienschmidt]

We need the following information:

• Your Go-MySQL-Driver version (or git SHA)
• Your Go version (run go version in your console)

[mejinke]

go version go1.1.2 darwin/amd64
Go-MySQL-Driver is latest version

[julienschmidt]

I think the panic shouldn't occur with v1.0.3, which is a bugfix release we released yesterday (go get-able since a few minutes).

But I'm investigating further, the stacktrace looks really strange to me. The panic is caused by a nil-connection in stmt.Close (variable mc) where a connection should never be nil with the above code.

v1.0.3 can at least prevent the nil-pointer panic, since it checks mc == nil first.

[julienschmidt]

Can you reproduce this error?

[mejinke]

panic: runtime error: invalid memory address or nil pointer dereference
[signal 0xb code=0x1 addr=0x41 pc=0x5304bc]
goroutine 25466 [running]:
github.com/go-sql-driver/mysql.(*mysqlConn).writeCommandPacketUint32(0x0, 0x305e92619, 0xc22b2fe650, 0x2b8b3c6167c0)
    /Users/jinke/golang/src/github.com/go-sql-driver/mysql/packets.go:327 +0x1c
github.com/go-sql-driver/mysql.(*mysqlStmt).Close(0xc22b87a6f0, 0x0, 0x0)
    /Users/jinke/golang/src/github.com/go-sql-driver/mysql/statement.go:24 +0x46
database/sql.(*driverConn).finalClose(0xc202a42780, 0xc2014fd1c0, 0x2210b5a80)
    /usr/local/go/src/pkg/database/sql/sql.go:Jump {0 autohome 25 44 258 1977 hkyjh 12.68 394 397 2011-9-27  http://k.autohome.com.cn/spec/8288/hkyjh 2013-11-02 14:21:38 6}
285 +0x87
database/sql.func·002(0xc2000ca200, 0xc2000aebd0)
    /usr/local/go/src/pkg/database/sql/sql.go:372 +0x2c
database/sql.(*driverConn).Close(0xc202a42780, 0x32, 0x9)
    /usr/local/go/src/pkg/database/sql/sql.go:278 +0x174
database/sql.(*DB).putConn(0xc2000ca1e0, 0xc202a42780, 0x0, 0x0)
    /usr/local/go/src/pkg/database/sql/sql.go:598 +0x2e2
database/sql.func·004()
    /usr/local/go/src/pkg/database/sql/sql.go:664 +0x41
database/sql.(*DB).exec(0xc2000ca1e0, 0x738330, 0x40, 0x2b8b3c616cb8, 0x2, ...)
    /usr/local/go/src/pkg/database/sql/sql.go:661 +0xb2
database/sql.(*DB).Exec(0xc2000ca1e0, 0x738330, 0x40, 0x2b8b3c616cb8, 0x2, ...)
    /usr/local/go/src/pkg/database/sql/sql.go:650 +0x98
cds_spider/newCar/koubei.AddOk(0x6dee10, 0x8, 0x1, 0x0, 0x0, ...)
    /Users/jinke/golang/src/cds_spider/newCar/koubei/log.go:49 +0x568
cds_spider/newCar/koubei.(*KouBei).Save(0xc20a0a74d0, 0x0, 0x0, 0x0)
    /Users/jinke/golang/src/cds_spider/newCar/koubei/koubei.go:29 +0x3b7
main.func·006()
    /Users/jinke/golang/src/cds_spider/newCar/koubei/main/autohome.go:241 +0x37
created by main.func·007

/Users/jinke/golang/src/cds_spider/newCar/koubei/log.go

func AddOk(source string, count int) error {
    row := dbpool.DB.QueryRow("SELECT * FROM cheduoshao_spider_status.koubei WHERE source=? AND spider_date=?", source, gcode.Date())
    var (
        id, quantity         int
        sources, spider_date string
    )

    row.Scan(&id, &sources, &quantity, &spider_date)

    if id == 0 {
        stmt, err := dbpool.DB.Prepare("INSERT cheduoshao_spider_status.koubei SET source=?, quantity=?, spider_date=?")
        if err != nil {
            return err
        }
        defer stmt.Close()

        _, err = stmt.Exec(source, 1, gcode.Date())
        if err != nil {
            return err
        }
    } else { 
        dbpool.DB.Exec("UPDATE main.koubei SET quantity=? WHERE id=?", quantity+1, id)
    }

    return nil
}

dbpool.DB.Exec("UPDATE main.koubei SET quantity=? WHERE id=?", quantity+1, id) IS ERROR

[mejinke]

Seem to have not found such a situation before, my system is OS X 10.9

[julienschmidt]

Check if row.Scan(&id, &sources, &quantity, &spider_date) returns an error.
Did you try v1.0.3? Like I said, it should at least avoid the panics.

[mejinke]

I tried a few times more, I just delete the old version, go get it again
I changed to:

err := row.Scan(&id, &sources, &quantity, &spider_date)
    if err != nil {
        return err
    }

[julienschmidt]

I think I was able to reproduce this bug. Once again, this bug is related to https://codereview.appspot.com/14920046 / Go issue 5718 (or #98, #142).
stmt.Exec ignores errors by the driver and puts bad connections back to the pool.

v1.0.3 includes #143, which at least avoids the panics. v1.1, which will be released in a few hours, will include #151, which improves this a bit more.

I'd suggest to either manually patch your Go version with https://codereview.appspot.com/14920046 or avoid stmt.Exec until the next stable release with this fix included is released. That would probably be Go 1.3 in June 2014...

And a note to your code above:
If you use a stmt only once, there is no reason to manually prepare it. Just use db.Exec, which is by the way not bugged like stmt.Exec.

Here is how I would modify your code above:

func AddOk(source string, count int) error {
    var (
        id, quantity         int
        sources, spider_date string
    )
    if err := dbpool.DB.QueryRow("SELECT * FROM cheduoshao_spider_status.koubei WHERE source=? AND spider_date=?", source, gcode.Date()).Scan(&id, &sources, &quantity, &spider_date); err != nil {
        return err
    }

    if id == 0 {
        if _, err =: dbpool.DB.Exec("INSERT cheduoshao_spider_status.koubei SET source=?, quantity=?, spider_date=?", source, 1, gcode.Date()); err != nil {
            return err
        }
    } else { 
        if _, err =: dbpool.DB.Exec("UPDATE main.koubei SET quantity=? WHERE id=?", quantity+1, id); err != nil {
            return err
        }
    }

    return nil
}

[mejinke]

Ok,Thanks!

[danilopolani]

I'm having too this problem, go version is go1.7.4 linux/amd64 and go-sql-driver is the latest. I'm connecting to a RDS instance of AWS.

runtime error: invalid memory address or nil pointer dereference
/usr/local/go/src/runtime/panic.go:458 (0x42c653)
	gopanic: reflectcall(nil, unsafe.Pointer(d.fn), deferArgs(d), uint32(d.siz), uint32(d.siz))
/usr/local/go/src/runtime/panic.go:62 (0x42b1ad)
	panicmem: panic(memoryError)
/usr/local/go/src/runtime/sigpanic_unix.go:24 (0x441054)
	sigpanic: panicmem()
/usr/local/go/src/database/sql/sql.go:781 (0x557d0a)
	(*DB).conn: db.mu.Lock()
/usr/local/go/src/database/sql/sql.go:1074 (0x559fbb)
	(*DB).query: ci, err := db.conn(strategy)
/usr/local/go/src/database/sql/sql.go:1062 (0x559dc0)
	(*DB).Query: rows, err = db.query(query, args, cachedOrNewConn)
/home/danilo/Go/src/github.com/DaniloMeritocracy/autocomplete/autocomplete.go:192 (0x402e27)
	_getJSON: rows, err := db.Query(sqlString)
/home/danilo/Go/src/github.com/DaniloMeritocracy/autocomplete/autocomplete.go:130 (0x402b54)
	school: res, err := _getJSON("SELECT * FROM university WHERE LOWER(name) LIKE '" + strings.Replace(term, "'", "\\'", -1) + "%' LIMIT 15")
/home/danilo/Go/src/github.com/DaniloMeritocracy/autocomplete/autocomplete.go:56 (0x403985)
	main.func1: results = school(term)
/home/danilo/Go/src/gopkg.in/gin-gonic/gin.v1/context.go:97 (0x45d53a)
	(*Context).Next: c.handlers[c.index](c)
/home/danilo/Go/src/gopkg.in/gin-gonic/gin.v1/recovery.go:45 (0x46d31a)
	RecoveryWithWriter.func1: c.Next()
/home/danilo/Go/src/gopkg.in/gin-gonic/gin.v1/context.go:97 (0x45d53a)
	(*Context).Next: c.handlers[c.index](c)
/home/danilo/Go/src/gopkg.in/gin-gonic/gin.v1/logger.go:72 (0x46c40f)
	LoggerWithWriter.func1: c.Next()
/home/danilo/Go/src/gopkg.in/gin-gonic/gin.v1/context.go:97 (0x45d53a)
	(*Context).Next: c.handlers[c.index](c)
/home/danilo/Go/src/gopkg.in/gin-gonic/gin.v1/gin.go:284 (0x4638ce)
	(*Engine).handleHTTPRequest: context.Next()
/home/danilo/Go/src/gopkg.in/gin-gonic/gin.v1/gin.go:265 (0x4631b0)
	(*Engine).ServeHTTP: engine.handleHTTPRequest(c)
/usr/local/go/src/net/http/server.go:2202 (0x4e1fdd)
	serverHandler.ServeHTTP: handler.ServeHTTP(rw, req)
/usr/local/go/src/net/http/server.go:1579 (0x4de947)
	(*conn).serve: serverHandler{c.server}.ServeHTTP(w, w.req)
/usr/local/go/src/runtime/asm_amd64.s:2086 (0x45c0f1)
	goexit: BYTE	$0x90	// NOP

This is the function _getJSON, that returns a JSON-Encoded string of the results:

func _getJSON(sqlString string) (string, error) {
	rows, err := db.Query(sqlString)
	if err != nil {
		return "", err
	}
	defer rows.Close()
	columns, err := rows.Columns()
	if err != nil {
		return "", err
	}
	count := len(columns)
	tableData := make([]map[string]interface{}, 0)
	values := make([]interface{}, count)
	valuePtrs := make([]interface{}, count)
	for rows.Next() {
		for i := 0; i < count; i++ {
			valuePtrs[i] = &values[i]
		}
		rows.Scan(valuePtrs...)
		entry := make(map[string]interface{})
		for i, col := range columns {
			var v interface{}
			val := values[i]
			b, ok := val.([]byte)
			if ok {
				v = string(b)
			} else {
				v = val
			}
			entry[col] = v
		}
		tableData = append(tableData, entry)
	}
	jsonData, err := json.Marshal(tableData)
	if err != nil {
		return "", err
	}
	//fmt.Println(string(jsonData))
	return string(jsonData), nil
}

[methane]

@danilopolani Your db is nil.

[danilopolani]

Why should it be nil? I have a db var outside the functions var db *sql.DB and in the main func I opened the connection with

db, err := sql.Open("mysql", "<USERNAME>:<PASSWORD>@tcp(<HOST>:3306)/<DB_NAME>")
	if err != nil {
		panic(err.Error())
	}
	defer db.Close()

Then it is called in _getJSON(). Credentials are OK. Maybe it's an AWS problem? I'll try with localhost.

[methane]

db, err := sql.Open("mysql", "<USERNAME>:<PASSWORD>@tcp(<HOST>:3306)/<DB_NAME>")

Then, db is local variable of main, not global variable _getJSON() using.

Without you provide complete reproducible code, no one can point out
where your code have race condition.
go build -race is your friend.