diff --git a/README.md b/README.md index 6acf6a5..de3a3e2 100644 --- a/README.md +++ b/README.md @@ -5,3 +5,7 @@ MySQL's LOAD DATA INFILE. More information can be found at https://dev.mysql.com/doc/refman/8.0/en/load-data.html#load-data-field-line-handling + +## Character sets + +Characters sets are the worst. Make sure to verify your data is loaded correctly before relying on this not to corrupt your data. diff --git a/mysqltsv.go b/mysqltsv.go index 42218b5..09cecc1 100644 --- a/mysqltsv.go +++ b/mysqltsv.go @@ -14,7 +14,7 @@ import ( ) // Escaping explains the escaping this package uses for inclusion in a LOAD DATA INFILE statement. -const Escaping = `FIELDS TERMINATED BY '\t' OPTIONALLY ENCLOSED BY '"' ESCAPED BY '\\' LINES TERMINATED BY '\n' STARTING BY ''` +const Escaping = `CHARACTER SET binary FIELDS TERMINATED BY '\t' OPTIONALLY ENCLOSED BY '"' ESCAPED BY '\\' LINES TERMINATED BY '\n' STARTING BY ''` /* type Options struct { diff --git a/tests/go.mod b/tests/go.mod new file mode 100644 index 0000000..7324183 --- /dev/null +++ b/tests/go.mod @@ -0,0 +1,11 @@ +module github.com/hexon/mysqltsv/tests + +go 1.21.4 + +require ( + github.com/davecgh/go-spew v1.1.1 + github.com/go-sql-driver/mysql v1.7.1 + github.com/hexon/mysqltsv v0.1.0 +) + +replace github.com/hexon/mysqltsv => ../ diff --git a/tests/go.sum b/tests/go.sum new file mode 100644 index 0000000..8fc0102 --- /dev/null +++ b/tests/go.sum @@ -0,0 +1,6 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= +github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= +github.com/hexon/mysqltsv v0.1.0 h1:48wYQlsPH8ZEkKAVCdsOYzMYAlEoevw8ZWD8rqYPdlg= +github.com/hexon/mysqltsv v0.1.0/go.mod h1:p3vPBkpxebjHWF1bWKYNcXx5pFu+yAG89QZQEKSvVrY= diff --git a/tests/roundtrip_test.go b/tests/roundtrip_test.go new file mode 100644 index 0000000..66da8c4 --- /dev/null +++ b/tests/roundtrip_test.go @@ -0,0 +1,109 @@ +package mysqltsv_test + +import ( + "bytes" + "context" + "database/sql" + "fmt" + "io" + "math/rand" + "os" + "testing" + + "github.com/davecgh/go-spew/spew" + "github.com/go-sql-driver/mysql" + "github.com/hexon/mysqltsv" +) + +var schema = ` +CREATE TEMPORARY TABLE roundtrip_test ( + id INT NOT NULL PRIMARY KEY, + data BLOB NOT NULL +); +` + +func TestRoundtrip(t *testing.T) { + ctx := context.Background() + dsn := os.Getenv("TEST_DSN") + if dsn == "" { + t.Fatalf("Environment variable TEST_DSN is empty") + } + db, err := sql.Open("mysql", dsn) + if err != nil { + t.Fatalf("Failed to connect to database: %v", err) + } + db.SetMaxOpenConns(1) + + if _, err := db.ExecContext(ctx, schema); err != nil { + t.Fatalf("Failed to create table: %v", err) + } + + var dataRows [][]byte + for i := 0; 1000 > i; i++ { + row := make([]byte, rand.Intn(2048)) + for j := range row { + row[j] = uint8(rand.Intn(255)) + } + dataRows = append(dataRows, row) + } + + var buf bytes.Buffer + e := mysqltsv.NewEncoder(&buf, 2, nil) + for i, row := range dataRows { + e.AppendValue(i) + e.AppendBytes(row) + } + if err := e.Close(); err != nil { + t.Fatalf("Encoding failed: %v", err) + } + + mysql.RegisterReaderHandler("buf", func() io.Reader { return &buf }) + res, err := db.ExecContext(ctx, fmt.Sprintf("LOAD DATA LOCAL INFILE 'Reader::buf' INTO TABLE `roundtrip_test` %s (id, data)", mysqltsv.Escaping)) + if err != nil { + t.Fatalf("LOAD DATA LOCAL INFILE failed: %v", err) + } + n, err := res.RowsAffected() + if err != nil { + t.Fatalf("RowsAffected failed: %v", err) + } + + rows, err := db.QueryContext(ctx, "SHOW WARNINGS") + if err != nil { + t.Fatalf("Failed to SHOW WARNINGS: %v", err) + } + for rows.Next() { + var level, code, message string + if err := rows.Scan(&level, &code, &message); err != nil { + t.Fatalf("Scan failed: %v", err) + } + t.Errorf("MySQL warning: %v", message) + } + if err := rows.Close(); err != nil { + t.Errorf("rows.Close: %v", err) + } + if n != int64(len(dataRows)) { + t.Fatalf("Tried to insert %d rows, but succeeded at only %d", len(dataRows), n) + } + + rows, err = db.QueryContext(ctx, "SELECT data FROM roundtrip_test ORDER BY id") + if err != nil { + t.Fatalf("Failed to read rows: %v", err) + } + + for rows.Next() { + var readData []byte + if err := rows.Scan(&readData); err != nil { + t.Fatalf("Scan failed: %v", err) + } + want := dataRows[0] + if !bytes.Equal(want, readData) { + t.Errorf("Mismatch!") + spew.Dump(want) + spew.Dump(readData) + } + dataRows = dataRows[1:] + } + if err := rows.Close(); err != nil { + t.Errorf("rows.Close: %v", err) + } +}