Insert fails for blobs >= 2^16 bytes

[nicktobey]

Depending on the exact method used, we can get two different stack traces and error messages.

Method 1: LOAD_FILE

repro steps:

dd if=/dev/zero of=blobfile bs=65536 count=1
dolt sql -q "create table files (pk int primary key, file blob);"
dolt sql -q "insert into files values (1, load_file('blobfile'));"

error: interface conversion: interface {} is []uint8, not string

Stack trace:

/Users/nick/go/pkg/mod/golang.org/[email protected]/src/runtime/iface.go:275 (0x100c34627)
	panicdottypeE: panic(&TypeAssertionError{iface, have, want, ""})
/Users/nick/go/pkg/mod/github.com/dolthub/[email protected]/sql/rowexec/insert.go:327 (0x10238d61b)
	com/dolthub/go-mysql-server/sql/rowexec.convertDataAndWarn: row[columnIdx] = row[columnIdx].(string)[:maxLength] // truncate string
/Users/nick/go/pkg/mod/github.com/dolthub/[email protected]/sql/rowexec/insert.go:118 (0x10238ca2b)
	com/dolthub/go-mysql-server/sql/rowexec.(*insertIter).Next: row = convertDataAndWarn(ctx, i.schema, row, idx, cErr)
/Users/nick/go/pkg/mod/github.com/dolthub/[email protected]/sql/plan/table_editor.go:121 (0x102102e9f)
	com/dolthub/go-mysql-server/sql/plan.CheckpointingTableEditorIter.Next: row, err := c.inner.Next(ctx)
/Users/nick/go/pkg/mod/github.com/dolthub/[email protected]/sql/rowexec/dml_iters.go:606 (0x102389aeb)
	com/dolthub/go-mysql-server/sql/rowexec.(*accumulatorIter).Next: row, err := a.iter.Next(ctx)
/Users/nick/go/pkg/mod/github.com/dolthub/[email protected]/sql/rowexec/transaction_iters.go:86 (0x1023bd627)
	com/dolthub/go-mysql-server/sql/rowexec.(*TransactionCommittingIter).Next: return t.childIter.Next(ctx)
/Users/nick/go/pkg/mod/github.com/dolthub/[email protected]/sql/plan/process.go:306 (0x1020dc50f)
	com/dolthub/go-mysql-server/sql/plan.(*TrackedRowIter).Next: row, err := i.iter.Next(ctx)
/Users/nick/go/pkg/mod/github.com/dolthub/[email protected]/server/handler.go:489 (0x1027267c7)
	com/dolthub/go-mysql-server/server.resultForOkIter: row, err := iter.Next(ctx)
/Users/nick/go/pkg/mod/github.com/dolthub/[email protected]/server/handler.go:447 (0x1027261c3)
	com/dolthub/go-mysql-server/server.(*Handler).doQuery: r, err = resultForOkIter(sqlCtx, rowIter)
/Users/nick/go/pkg/mod/github.com/dolthub/[email protected]/server/handler.go:759 (0x102728dd7)
	com/dolthub/go-mysql-server/server.(*Handler).errorWrappedDoQuery: remainder, err := h.doQuery(ctx, c, query, parsed, nil, mode, h.executeQuery, bindings, callback, qFlags)
/Users/nick/go/pkg/mod/github.com/dolthub/[email protected]/server/handler.go:310 (0x1027254d3)
	com/dolthub/go-mysql-server/server.(*Handler).ComQuery: _, err := h.errorWrappedDoQuery(ctx, c, query, nil, MultiStmtModeOff, nil, callback, &sql.QueryFlags{})
/Users/nick/go/pkg/mod/github.com/dolthub/[email protected]/go/mysql/conn.go:1569 (0x10118af73)
	com/dolthub/vitess/go/mysql.(*Conn).execQuery: err = handler.ComQuery(ctx, c, query, resultsCB)
/Users/nick/go/pkg/mod/github.com/dolthub/[email protected]/go/mysql/conn.go:1002 (0x101186763)
	com/dolthub/vitess/go/mysql.(*Conn).handleNextCommand: for query, err = c.execQuery(ctx, query, handler, multiStatements); err == nil && query != ""; {
/Users/nick/go/pkg/mod/github.com/dolthub/[email protected]/go/mysql/server.go:524 (0x1011a16db)
	com/dolthub/vitess/go/mysql.(*Listener).handle: err := c.handleNextCommand(ctx, l.handler)
/Users/nick/go/pkg/mod/golang.org/[email protected]/src/runtime/asm_arm64.s:1223 (0x100ca6103)
	goexit: MOVD	R0, R0	// NOP

Method 2: inline file in query

repro.py:

import pymysql

db = pymysql.connect(unix_socket="/tmp/mysql.sock", user="root", database="repro")
cur = db.cursor()

blob = b'x' * 65536
cur.execute("insert into files values (1, _binary %s)", (blob,))
cur.execute("commit")

error: interface conversion: interface {} is string, not []uint8

stack trace:

/Users/nick/go/pkg/mod/golang.org/[email protected]/src/runtime/iface.go:275 (0x100c34627)
	panicdottypeE: panic(&TypeAssertionError{iface, have, want, ""})
/Users/nick/Documents/dolt/go/store/prolly/tree/prolly_fields.go:263 (0x101ae4b0f)
	com/dolthub/dolt/go/store/prolly/tree.PutField: h, err := SerializeBytesToAddr(ctx, ns, bytes.NewReader(v.([]byte)), len(v.([]byte)))
/Users/nick/Documents/dolt/go/libraries/doltcore/sqle/writer/prolly_index_writer.go:148 (0x102760313)
	com/dolthub/dolt/go/libraries/doltcore/sqle/writer.prollyIndexWriter.Insert: if err := tree.PutField(ctx, m.mut.NodeStore(), m.valBld, to, sqlRow[from]); err != nil {
/Users/nick/Documents/dolt/go/libraries/doltcore/sqle/writer/prolly_table_writer.go:160 (0x102766dff)
	com/dolthub/dolt/go/libraries/doltcore/sqle/writer.(*prollyTableWriter).Insert: if err = w.primary.Insert(ctx, sqlRow); err != nil {
/Users/nick/go/pkg/mod/github.com/dolthub/[email protected]/sql/rowexec/insert.go:164 (0x10238c40f)
	com/dolthub/go-mysql-server/sql/rowexec.(*insertIter).Next: if err := i.inserter.Insert(ctx, row); err != nil {
/Users/nick/go/pkg/mod/github.com/dolthub/[email protected]/sql/plan/table_editor.go:121 (0x102102e9f)
	com/dolthub/go-mysql-server/sql/plan.CheckpointingTableEditorIter.Next: row, err := c.inner.Next(ctx)
/Users/nick/go/pkg/mod/github.com/dolthub/[email protected]/sql/rowexec/dml_iters.go:606 (0x102389aeb)
	com/dolthub/go-mysql-server/sql/rowexec.(*accumulatorIter).Next: row, err := a.iter.Next(ctx)
/Users/nick/go/pkg/mod/github.com/dolthub/[email protected]/sql/rowexec/transaction_iters.go:86 (0x1023bd627)
	com/dolthub/go-mysql-server/sql/rowexec.(*TransactionCommittingIter).Next: return t.childIter.Next(ctx)
/Users/nick/go/pkg/mod/github.com/dolthub/[email protected]/sql/plan/process.go:306 (0x1020dc50f)
	com/dolthub/go-mysql-server/sql/plan.(*TrackedRowIter).Next: row, err := i.iter.Next(ctx)
/Users/nick/go/pkg/mod/github.com/dolthub/[email protected]/server/handler.go:489 (0x1027267c7)
	com/dolthub/go-mysql-server/server.resultForOkIter: row, err := iter.Next(ctx)
/Users/nick/go/pkg/mod/github.com/dolthub/[email protected]/server/handler.go:447 (0x1027261c3)
	com/dolthub/go-mysql-server/server.(*Handler).doQuery: r, err = resultForOkIter(sqlCtx, rowIter)
/Users/nick/go/pkg/mod/github.com/dolthub/[email protected]/server/handler.go:759 (0x102728dd7)
	com/dolthub/go-mysql-server/server.(*Handler).errorWrappedDoQuery: remainder, err := h.doQuery(ctx, c, query, parsed, nil, mode, h.executeQuery, bindings, callback, qFlags)
/Users/nick/go/pkg/mod/github.com/dolthub/[email protected]/server/handler.go:310 (0x1027254d3)
	com/dolthub/go-mysql-server/server.(*Handler).ComQuery: _, err := h.errorWrappedDoQuery(ctx, c, query, nil, MultiStmtModeOff, nil, callback, &sql.QueryFlags{})
/Users/nick/go/pkg/mod/github.com/dolthub/[email protected]/go/mysql/conn.go:1569 (0x10118af73)
	com/dolthub/vitess/go/mysql.(*Conn).execQuery: err = handler.ComQuery(ctx, c, query, resultsCB)
/Users/nick/go/pkg/mod/github.com/dolthub/[email protected]/go/mysql/conn.go:1002 (0x101186763)
	com/dolthub/vitess/go/mysql.(*Conn).handleNextCommand: for query, err = c.execQuery(ctx, query, handler, multiStatements); err == nil && query != ""; {
/Users/nick/go/pkg/mod/github.com/dolthub/[email protected]/go/mysql/server.go:524 (0x1011a16db)
	com/dolthub/vitess/go/mysql.(*Listener).handle: err := c.handleNextCommand(ctx, l.handler)
/Users/nick/go/pkg/mod/golang.org/[email protected]/src/runtime/asm_arm64.s:1223 (0x100ca6103)
	goexit: MOVD	R0, R0	// NOP

Note that the errors, while similar, are different: interface conversion: interface {} is []uint8, not string vs interface conversion: interface {} is string, not []uint8

[nicktobey]

So the reason for the failure makes sense: the BLOB type mirrors MySQL's BLOB type and maxes out at 65536 bytes. For longer blobs, we have the MEDIUMBLOB and LARGEBLOB types.

However, this should be an error, not a panic.

Oddly, as of today I'm unable to reproduce the panic on my dev machine with the above repro steps, instead getting the appropriate error.

My windows machine where I first discovered the panic can still reproduce it. Both machines have the same version installed. I'm not yet sure why this changed.