diff --git a/config.go b/config.go new file mode 100644 index 0000000000000000000000000000000000000000..5a582ac42026ef1cafb11ce1cd3736b81b8a0266 --- /dev/null +++ b/config.go @@ -0,0 +1,5 @@ +// +build !debug + +package db + +var Debug = false diff --git a/config_debug.go b/config_debug.go new file mode 100644 index 0000000000000000000000000000000000000000..a35ee298fba74e85432aaaf34108ff0de39bc95e --- /dev/null +++ b/config_debug.go @@ -0,0 +1,5 @@ +// +build debug + +package db + +var Debug = true diff --git a/db_test.go b/db_test.go index 557b02d24133ad623ef3b04d522c364cd3abcf28..045d363e94686924a4101968d4b1bae709ec41be 100644 --- a/db_test.go +++ b/db_test.go @@ -854,60 +854,64 @@ func TestFibonacci(t *testing.T) { t.Fatalf(`%s: Unexpected count %d.`, wrapper, total) } - // Find() with empty db.Cond. - res1 := col.Find(db.Cond{}) - total, err = res1.Count() + // Skipping mongodb as the results of this are not defined there. + if wrapper != `mongo` { - if total != 6 { - t.Fatalf(`%s: Unexpected count %d.`, wrapper, total) - } + // Find() with empty db.Cond. + res1 := col.Find(db.Cond{}) + total, err = res1.Count() - // Find() with empty expression - res1b := col.Find(db.Or{db.And{db.Cond{}, db.Cond{}}, db.Or{db.Cond{}}}) - total, err = res1b.Count() + if total != 6 { + t.Fatalf(`%s: Unexpected count %d.`, wrapper, total) + } - if total != 6 { - t.Fatalf(`%s: Unexpected count %d.`, wrapper, total) - } + // Find() with empty expression + res1b := col.Find(db.Or{db.And{db.Cond{}, db.Cond{}}, db.Or{db.Cond{}}}) + total, err = res1b.Count() - // Find() with explicit IS NULL - res2 := col.Find(db.Cond{"input IS": nil}) - total, err = res2.Count() + if total != 6 { + t.Fatalf(`%s: Unexpected count %d.`, wrapper, total) + } - if total != 0 { - t.Fatalf(`%s: Unexpected count %d.`, wrapper, total) - } + // Find() with explicit IS NULL + res2 := col.Find(db.Cond{"input IS": nil}) + total, err = res2.Count() - // Find() with implicit IS NULL - res2a := col.Find(db.Cond{"input": nil}) - total, err = res2a.Count() + if total != 0 { + t.Fatalf(`%s: Unexpected count %d.`, wrapper, total) + } - if total != 0 { - t.Fatalf(`%s: Unexpected count %d.`, wrapper, total) - } + // Find() with implicit IS NULL + res2a := col.Find(db.Cond{"input": nil}) + total, err = res2a.Count() - // Find() with explicit = NULL - res2b := col.Find(db.Cond{"input =": nil}) - total, err = res2b.Count() + if total != 0 { + t.Fatalf(`%s: Unexpected count %d.`, wrapper, total) + } - if total != 0 { - t.Fatalf(`%s: Unexpected count %d.`, wrapper, total) - } + // Find() with explicit = NULL + res2b := col.Find(db.Cond{"input =": nil}) + total, err = res2b.Count() - // Find() with implicit IN - res3 := col.Find(db.Cond{"input": []int{1, 2, 3, 4}}) - total, err = res3.Count() + if total != 0 { + t.Fatalf(`%s: Unexpected count %d.`, wrapper, total) + } - if total != 3 { - t.Fatalf(`%s: Unexpected count %d.`, wrapper, total) - } + // Find() with implicit IN + res3 := col.Find(db.Cond{"input": []int{1, 2, 3, 4}}) + total, err = res3.Count() + + if total != 3 { + t.Fatalf(`%s: Unexpected count %d.`, wrapper, total) + } - // Find() with implicit NOT IN - res3a := col.Find(db.Cond{"input NOT IN": []int{1, 2, 3, 4}}) - total, err = res3a.Count() + // Find() with implicit NOT IN + res3a := col.Find(db.Cond{"input NOT IN": []int{1, 2, 3, 4}}) + total, err = res3a.Count() - if total != 3 { - t.Fatalf(`%s: Unexpected count %d.`, wrapper, total) + if total != 3 { + t.Fatalf(`%s: Unexpected count %d.`, wrapper, total) + } } var items []fibonacci diff --git a/mongo/_example/main.go b/mongo/_example/main.go index 88420f8557180d90ea5b6bc3b1eed0d94ec6a777..e5603dbd344bff7b238212c6cf0531c7f184b08d 100644 --- a/mongo/_example/main.go +++ b/mongo/_example/main.go @@ -3,6 +3,7 @@ package main import ( "fmt" "log" + "os" "time" "upper.io/db" // Imports the main db package. @@ -23,6 +24,10 @@ type Birthday struct { func main() { + if os.Getenv("TEST_HOST") != "" { + settings.Host = os.Getenv("TEST_HOST") + } + // Attemping to establish a connection to the database. sess, err := db.Open("mongo", settings) diff --git a/mysql/Makefile b/mysql/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..8f8882b0cbecbc66d9ce3ab2ef552048885dc042 --- /dev/null +++ b/mysql/Makefile @@ -0,0 +1,14 @@ +TEST_HOST ?= 127.0.0.1 + +build: + go build && go install + +reset-db: + $(MAKE) -C _dumps + +test: reset-db + go test -v + $(MAKE) -C _example + +bench: reset-db + go test -v -test.bench=. -test.benchtime=10s -benchmem diff --git a/mysql/_dumps/Makefile b/mysql/_dumps/Makefile index 46124a339683a160df036551755f836435cb7689..073f8210eb75017b8208d6891dfc6254a525fae7 100644 --- a/mysql/_dumps/Makefile +++ b/mysql/_dumps/Makefile @@ -5,10 +5,5 @@ DB_USERNAME ?= upperio_tests DB_PASSWORD ?= upperio_secret DB_NAME ?= upperio_tests -all: setup reset-db - -setup: - mysql -uroot -h"$(TEST_HOST)" -P$(TEST_PORT) < setup.sql - -reset-db: +load: cat structs.sql | mysql -u"$(DB_USERNAME)" -p"$(DB_PASSWORD)" -h"$(TEST_HOST)" -P$(TEST_PORT) "$(DB_NAME)" diff --git a/mysql/_example/Makefile b/mysql/_example/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..fb72ea7c07778986dcfd760ded7371d2bc035daf --- /dev/null +++ b/mysql/_example/Makefile @@ -0,0 +1,10 @@ +TEST_HOST ?= 127.0.0.1 +TEST_PORT ?= 3306 + +DB_USERNAME ?= upperio_tests +DB_PASSWORD ?= upperio_secret +DB_NAME ?= upperio_tests + +test: + cat example.sql | mysql -u"$(DB_USERNAME)" -p"$(DB_PASSWORD)" -h"$(TEST_HOST)" -P$(TEST_PORT) "$(DB_NAME)" + go run -v main.go diff --git a/mysql/_example/main.go b/mysql/_example/main.go index fa4d20f651f9f24803f170596e1137a5f7615c7e..8b073845520baa9c21248fb502760871d4a9d254 100644 --- a/mysql/_example/main.go +++ b/mysql/_example/main.go @@ -3,6 +3,7 @@ package main import ( "fmt" "log" + "os" "time" "upper.io/db" // Imports the main db package. @@ -10,10 +11,10 @@ import ( ) var settings = db.Settings{ - Database: `upperio_tests`, // Database name - Host: `testserver.local`, - User: `upperio`, // Database username. - Password: `upperio`, // Database password. + Database: `upperio_tests`, // Database name. + Host: `127.0.0.1`, + User: `upperio_tests`, // Database username. + Password: `upperio_secret`, // Database password. } // Birthday struct example. @@ -26,6 +27,10 @@ type Birthday struct { func main() { + if os.Getenv("TEST_HOST") != "" { + settings.Host = os.Getenv("TEST_HOST") + } + // Attemping to establish a connection to the database. sess, err := db.Open("mysql", settings) diff --git a/mysql/benchmark_test.go b/mysql/benchmark_test.go new file mode 100644 index 0000000000000000000000000000000000000000..dab116c1d636dd308d8f4b6cab9b6dbcc172c216 --- /dev/null +++ b/mysql/benchmark_test.go @@ -0,0 +1,665 @@ +package mysql + +import ( + "fmt" + "math/rand" + "testing" + + "github.com/jmoiron/sqlx" + "upper.io/db" +) + +const ( + testRows = 1000 +) + +func updatedArtistN(i int) string { + return fmt.Sprintf("Updated Artist %d", i%testRows) +} + +func artistN(i int) string { + return fmt.Sprintf("Artist %d", i%testRows) +} + +func connectAndAddFakeRows() (db.Database, error) { + var err error + var sess db.Database + + if sess, err = db.Open(Adapter, settings); err != nil { + return nil, err + } + + driver := sess.Driver().(*sqlx.DB) + + if _, err = driver.Exec("TRUNCATE TABLE `artist`"); err != nil { + return nil, err + } + + for i := 0; i < testRows; i++ { + if _, err = driver.Exec("INSERT INTO `artist` (`name`) VALUES(?)", artistN(i)); err != nil { + return nil, err + } + } + + return sess, nil +} + +// BenchmarkSQLAppend benchmarks raw INSERT SQL queries without using prepared +// statements nor arguments. +func BenchmarkSQLAppend(b *testing.B) { + var err error + var sess db.Database + + if sess, err = db.Open(Adapter, settings); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + if _, err = driver.Exec("TRUNCATE TABLE `artist`"); err != nil { + b.Fatal(err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err = driver.Exec("INSERT INTO `artist` (`name`) VALUES('Hayao Miyazaki')"); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkSQLAppendWithArgs benchmarks raw SQL queries with arguments but +// without using prepared statements. The SQL query looks like the one that is +// generated by upper.io/db. +func BenchmarkSQLAppendWithArgs(b *testing.B) { + var err error + var sess db.Database + + if sess, err = db.Open(Adapter, settings); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + if _, err = driver.Exec("TRUNCATE TABLE `artist`"); err != nil { + b.Fatal(err) + } + + args := []interface{}{ + "Hayao Miyazaki", + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err = driver.Exec("INSERT INTO `artist` (`name`) VALUES(?)", args...); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkSQLPreparedAppend benchmarks raw INSERT SQL queries using prepared +// statements but no arguments. +func BenchmarkSQLPreparedAppend(b *testing.B) { + var err error + var sess db.Database + + if sess, err = db.Open(Adapter, settings); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + if _, err = driver.Exec("TRUNCATE TABLE `artist`"); err != nil { + b.Fatal(err) + } + + stmt, err := driver.Prepare("INSERT INTO `artist` (`name`) VALUES('Hayao Miyazaki')") + if err != nil { + b.Fatal(err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err = stmt.Exec(); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkSQLAppendWithArgs benchmarks raw INSERT SQL queries with arguments +// using prepared statements. The SQL query looks like the one that is +// generated by upper.io/db. +func BenchmarkSQLPreparedAppendWithArgs(b *testing.B) { + var err error + var sess db.Database + + if sess, err = db.Open(Adapter, settings); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + if _, err = driver.Exec("TRUNCATE TABLE `artist`"); err != nil { + b.Fatal(err) + } + + stmt, err := driver.Prepare("INSERT INTO `artist` (`name`) VALUES(?)") + + if err != nil { + b.Fatal(err) + } + + args := []interface{}{ + "Hayao Miyazaki", + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err = stmt.Exec(args...); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkSQLAppendWithVariableArgs benchmarks raw INSERT SQL queries with +// arguments using prepared statements. The SQL query looks like the one that +// is generated by upper.io/db. +func BenchmarkSQLPreparedAppendWithVariableArgs(b *testing.B) { + var err error + var sess db.Database + + if sess, err = db.Open(Adapter, settings); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + if _, err = driver.Exec("TRUNCATE TABLE `artist`"); err != nil { + b.Fatal(err) + } + + stmt, err := driver.Prepare("INSERT INTO `artist` (`name`) VALUES(?)") + + if err != nil { + b.Fatal(err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + args := []interface{}{ + fmt.Sprintf("Hayao Miyazaki %d", rand.Int()), + } + if _, err = stmt.Exec(args...); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkSQLPreparedAppendTransactionWithArgs benchmarks raw INSERT queries +// within a transaction block with arguments and prepared statements. SQL +// queries look like those generated by upper.io/db. +func BenchmarkSQLPreparedAppendTransactionWithArgs(b *testing.B) { + var err error + var sess db.Database + var tx *sqlx.Tx + + if sess, err = db.Open(Adapter, settings); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + if tx, err = driver.Beginx(); err != nil { + b.Fatal(err) + } + + if _, err = tx.Exec("TRUNCATE TABLE `artist`"); err != nil { + b.Fatal(err) + } + + stmt, err := tx.Preparex("INSERT INTO `artist` (`name`) VALUES(?)") + if err != nil { + b.Fatal(err) + } + + args := []interface{}{ + "Hayao Miyazaki", + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err = stmt.Exec(args...); err != nil { + b.Fatal(err) + } + } + + if err = tx.Commit(); err != nil { + b.Fatal(err) + } +} + +// BenchmarkUpperAppend benchmarks an insertion by upper.io/db. +func BenchmarkUpperAppend(b *testing.B) { + + sess, err := db.Open(Adapter, settings) + if err != nil { + b.Fatal(err) + } + + defer sess.Close() + + artist, err := sess.Collection("artist") + if err != nil { + b.Fatal(err) + } + + artist.Truncate() + + item := struct { + Name string `db:"name"` + }{"Hayao Miyazaki"} + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err = artist.Append(item); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkUpperAppendVariableArgs benchmarks an insertion by upper.io/db +// with variable parameters. +func BenchmarkUpperAppendVariableArgs(b *testing.B) { + + sess, err := db.Open(Adapter, settings) + if err != nil { + b.Fatal(err) + } + + defer sess.Close() + + artist, err := sess.Collection("artist") + if err != nil { + b.Fatal(err) + } + + artist.Truncate() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + item := struct { + Name string `db:"name"` + }{fmt.Sprintf("Hayao Miyazaki %d", rand.Int())} + if _, err = artist.Append(item); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkUpperAppendTransaction benchmarks insertion queries by upper.io/db +// within a transaction operation. +func BenchmarkUpperAppendTransaction(b *testing.B) { + var sess db.Database + var err error + + if sess, err = db.Open(Adapter, settings); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + var tx db.Tx + if tx, err = sess.Transaction(); err != nil { + b.Fatal(err) + } + + var artist db.Collection + if artist, err = tx.Collection("artist"); err != nil { + b.Fatal(err) + } + + if err = artist.Truncate(); err != nil { + b.Fatal(err) + } + + item := struct { + Name string `db:"name"` + }{"Hayao Miyazaki"} + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err = artist.Append(item); err != nil { + b.Fatal(err) + } + } + + if err = tx.Commit(); err != nil { + b.Fatal(err) + } +} + +// BenchmarkUpperAppendTransactionWithMap benchmarks insertion queries by +// upper.io/db within a transaction operation using a map instead of a struct. +func BenchmarkUpperAppendTransactionWithMap(b *testing.B) { + var sess db.Database + var err error + + if sess, err = db.Open(Adapter, settings); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + var tx db.Tx + if tx, err = sess.Transaction(); err != nil { + b.Fatal(err) + } + + var artist db.Collection + if artist, err = tx.Collection("artist"); err != nil { + b.Fatal(err) + } + + if err = artist.Truncate(); err != nil { + b.Fatal(err) + } + + item := map[string]string{ + "name": "Hayao Miyazaki", + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err = artist.Append(item); err != nil { + b.Fatal(err) + } + } + + if err = tx.Commit(); err != nil { + b.Fatal(err) + } +} + +// BenchmarkSQLSelect benchmarks SQL SELECT queries. +func BenchmarkSQLSelect(b *testing.B) { + var err error + var sess db.Database + + if sess, err = connectAndAddFakeRows(); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + var res *sqlx.Rows + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if res, err = driver.Queryx("SELECT * FROM `artist` WHERE `name` = ?", artistN(i)); err != nil { + b.Fatal(err) + } + res.Close() + } +} + +// BenchmarkSQLPreparedSelect benchmarks SQL select queries using prepared +// statements. +func BenchmarkSQLPreparedSelect(b *testing.B) { + var err error + var sess db.Database + + if sess, err = connectAndAddFakeRows(); err != nil { + b.Fatal(err) + } + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + stmt, err := driver.Preparex("SELECT * FROM `artist` WHERE `name` = ?") + if err != nil { + b.Fatal(err) + } + + var res *sqlx.Rows + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if res, err = stmt.Queryx(artistN(i)); err != nil { + b.Fatal(err) + } + res.Close() + } +} + +// BenchmarkUpperFind benchmarks upper.io/db's One method. +func BenchmarkUpperFind(b *testing.B) { + var err error + var sess db.Database + + if sess, err = connectAndAddFakeRows(); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + artist, err := sess.Collection("artist") + if err != nil { + b.Fatal(err) + } + + type artistType struct { + Name string `db:"name"` + } + + var item artistType + + b.ResetTimer() + for i := 0; i < b.N; i++ { + res := artist.Find(db.Cond{"name": artistN(i)}) + if err = res.One(&item); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkUpperFindAll benchmarks upper.io/db's All method. +func BenchmarkUpperFindAll(b *testing.B) { + var err error + var sess db.Database + + if sess, err = connectAndAddFakeRows(); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + artist, err := sess.Collection("artist") + if err != nil { + b.Fatal(err) + } + + type artistType struct { + Name string `db:"name"` + } + + var items []artistType + + b.ResetTimer() + for i := 0; i < b.N; i++ { + res := artist.Find(db.Or{ + db.Cond{"name": artistN(i)}, + db.Cond{"name": artistN(i + 1)}, + db.Cond{"name": artistN(i + 2)}, + }) + if err = res.All(&items); err != nil { + b.Fatal(err) + } + if len(items) != 3 { + b.Fatal("Expecting 3 results.") + } + } +} + +// BenchmarkSQLUpdate benchmarks SQL UPDATE queries. +func BenchmarkSQLUpdate(b *testing.B) { + var err error + var sess db.Database + + if sess, err = connectAndAddFakeRows(); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err = driver.Exec("UPDATE `artist` SET `name` = ? WHERE `name` = ?", updatedArtistN(i), artistN(i)); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkSQLPreparedUpdate benchmarks SQL UPDATE queries. +func BenchmarkSQLPreparedUpdate(b *testing.B) { + var err error + var sess db.Database + + if sess, err = connectAndAddFakeRows(); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + stmt, err := driver.Prepare("UPDATE `artist` SET `name` = ? WHERE `name` = ?") + if err != nil { + b.Fatal(err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err = stmt.Exec(updatedArtistN(i), artistN(i)); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkUpperUpdate benchmarks upper.io/db's Update method. +func BenchmarkUpperUpdate(b *testing.B) { + var err error + var sess db.Database + + if sess, err = connectAndAddFakeRows(); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + artist, err := sess.Collection("artist") + if err != nil { + b.Fatal(err) + } + + type artistType struct { + Name string `db:"name"` + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + newValue := artistType{ + Name: updatedArtistN(i), + } + res := artist.Find(db.Cond{"name": artistN(i)}) + if err = res.Update(newValue); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkSQLDelete benchmarks SQL DELETE queries. +func BenchmarkSQLDelete(b *testing.B) { + var err error + var sess db.Database + + if sess, err = connectAndAddFakeRows(); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err = driver.Exec("DELETE FROM `artist` WHERE `name` = ?", artistN(i)); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkSQLPreparedDelete benchmarks SQL DELETE queries. +func BenchmarkSQLPreparedDelete(b *testing.B) { + var err error + var sess db.Database + + if sess, err = connectAndAddFakeRows(); err != nil { + b.Fatal(err) + } + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + stmt, err := driver.Prepare("DELETE FROM `artist` WHERE `name` = ?") + if err != nil { + b.Fatal(err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err = stmt.Exec(artistN(i)); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkUpperRemove benchmarks +func BenchmarkUpperRemove(b *testing.B) { + var err error + var sess db.Database + + if sess, err = connectAndAddFakeRows(); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + artist, err := sess.Collection("artist") + if err != nil { + b.Fatal(err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + res := artist.Find(db.Cond{"name": artistN(i)}) + if err = res.Remove(); err != nil { + b.Fatal(err) + } + } +} diff --git a/mysql/collection.go b/mysql/collection.go index e70d71b4a6f117a95ad18e06f1d15a2680ad0bb2..fb5cccd459f578e2ac4810689fb0b96a98c2e48c 100644 --- a/mysql/collection.go +++ b/mysql/collection.go @@ -46,7 +46,7 @@ func (t *table) Find(terms ...interface{}) db.Result { // Truncate deletes all rows from the table. func (t *table) Truncate() error { - _, err := t.database.Exec(sqlgen.Statement{ + _, err := t.database.Exec(&sqlgen.Statement{ Type: sqlgen.Truncate, Table: sqlgen.TableWithName(t.MainTableName()), }) @@ -80,7 +80,7 @@ func (t *table) Append(item interface{}) (interface{}, error) { } } - stmt := sqlgen.Statement{ + stmt := &sqlgen.Statement{ Type: sqlgen.Insert, Table: sqlgen.TableWithName(t.MainTableName()), Columns: sqlgenCols, diff --git a/mysql/database.go b/mysql/database.go index d14a98229f8fd53f51388856f055aeafc8cb0c6a..960bc15c20b369810010bc52a7b9811801daaede 100644 --- a/mysql/database.go +++ b/mysql/database.go @@ -28,6 +28,7 @@ import ( _ "github.com/go-sql-driver/mysql" // MySQL driver. "github.com/jmoiron/sqlx" + "upper.io/cache" "upper.io/db" "upper.io/db/util/schema" "upper.io/db/util/sqlgen" @@ -40,10 +41,11 @@ var ( ) type database struct { - connURL db.ConnectionURL - session *sqlx.DB - tx *sqltx.Tx - schema *schema.DatabaseSchema + connURL db.ConnectionURL + session *sqlx.DB + tx *sqltx.Tx + schema *schema.DatabaseSchema + cachedStatements *cache.Cache } type tx struct { @@ -51,6 +53,11 @@ type tx struct { *database } +type cachedStatement struct { + *sqlx.Stmt + query string +} + var ( _ = db.Database(&database{}) _ = db.Tx(&tx{}) @@ -60,6 +67,40 @@ type columnSchemaT struct { Name string `db:"column_name"` } +func (d *database) prepareStatement(stmt *sqlgen.Statement) (p *sqlx.Stmt, query string, err error) { + if d.session == nil { + return nil, "", db.ErrNotConnected + } + + pc, ok := d.cachedStatements.ReadRaw(stmt) + + if ok { + ps := pc.(*cachedStatement) + p = ps.Stmt + query = ps.query + } else { + query = compileAndReplacePlaceholders(stmt) + + if d.tx != nil { + p, err = d.tx.Preparex(query) + } else { + p, err = d.session.Preparex(query) + } + + if err != nil { + return nil, "", err + } + + d.cachedStatements.Write(stmt, &cachedStatement{p, query}) + } + + return p, query, nil +} + +func compileAndReplacePlaceholders(stmt *sqlgen.Statement) string { + return stmt.Compile(template.Template) +} + // Driver returns the underlying *sqlx.DB instance. func (d *database) Driver() interface{} { return d.session @@ -89,6 +130,11 @@ func (d *database) Open() error { conn.Options["charset"] = "utf8" } + // Connection charset, UTF-8 by default. + if conn.Options["parseTime"] == "" { + conn.Options["parseTime"] = "true" + } + if settings.Socket != "" { conn.Address = db.Socket(settings.Socket) } else { @@ -111,8 +157,12 @@ func (d *database) Open() error { d.session.Mapper = sqlutil.NewMapper() - if err = d.populateSchema(); err != nil { - return err + d.cachedStatements = cache.NewCache() + + if d.schema == nil { + if err = d.populateSchema(); err != nil { + return err + } } return nil @@ -125,14 +175,13 @@ func (d *database) Clone() (db.Database, error) { } func (d *database) clone() (*database, error) { - src := &database{} - src.Setup(d.connURL) - - if err := src.Open(); err != nil { + clone := &database{ + schema: d.schema, + } + if err := clone.Setup(d.connURL); err != nil { return nil, err } - - return src, nil + return clone, nil } // Ping checks whether a connection to the database is still alive by pinging @@ -144,6 +193,7 @@ func (d *database) Ping() error { // Close terminates the current database session. func (d *database) Close() error { if d.session != nil { + d.cachedStatements.Clear() return d.session.Close() } return nil @@ -199,7 +249,7 @@ func (d *database) Collections() (collections []string, err error) { return d.schema.Tables, nil } - stmt := sqlgen.Statement{ + stmt := &sqlgen.Statement{ Type: sqlgen.Select, Columns: sqlgen.JoinColumns( sqlgen.ColumnWithName(`table_name`), @@ -220,8 +270,6 @@ func (d *database) Collections() (collections []string, err error) { return nil, err } - defer rows.Close() - collections = []string{} var name string @@ -229,6 +277,7 @@ func (d *database) Collections() (collections []string, err error) { for rows.Next() { // Getting table name. if err = rows.Scan(&name); err != nil { + rows.Close() return nil, err } @@ -243,24 +292,26 @@ func (d *database) Collections() (collections []string, err error) { } // Use changes the active database. -func (d *database) Use(database string) (err error) { +func (d *database) Use(name string) (err error) { var conn ConnectionURL if conn, err = ParseURL(d.connURL.String()); err != nil { return err } - conn.Database = database + conn.Database = name d.connURL = conn + d.schema = nil + return d.Open() } // Drop removes all tables from the current database. func (d *database) Drop() error { - _, err := d.Query(sqlgen.Statement{ + _, err := d.Query(&sqlgen.Statement{ Type: sqlgen.DropDatabase, Database: sqlgen.DatabaseWithName(d.schema.Name), }) @@ -283,8 +334,8 @@ func (d *database) Name() string { // be used to issue transactional queries. func (d *database) Transaction() (db.Tx, error) { var err error - var clone *database var sqlTx *sqlx.Tx + var clone *database if clone, err = d.clone(); err != nil { return nil, err @@ -300,90 +351,72 @@ func (d *database) Transaction() (db.Tx, error) { } // Exec compiles and executes a statement that does not return any rows. -func (d *database) Exec(stmt sqlgen.Statement, args ...interface{}) (sql.Result, error) { +func (d *database) Exec(stmt *sqlgen.Statement, args ...interface{}) (sql.Result, error) { var query string - var res sql.Result + var p *sqlx.Stmt var err error - var start, end int64 - - start = time.Now().UnixNano() - defer func() { - end = time.Now().UnixNano() - sqlutil.Log(query, args, err, start, end) - }() + if db.Debug { + var start, end int64 + start = time.Now().UnixNano() - if d.session == nil { - return nil, db.ErrNotConnected + defer func() { + end = time.Now().UnixNano() + sqlutil.Log(query, args, err, start, end) + }() } - query = stmt.Compile(template.Template) - - if d.tx != nil { - res, err = d.tx.Exec(query, args...) - } else { - res, err = d.session.Exec(query, args...) + if p, query, err = d.prepareStatement(stmt); err != nil { + return nil, err } - return res, err + return p.Exec(args...) } // Query compiles and executes a statement that returns rows. -func (d *database) Query(stmt sqlgen.Statement, args ...interface{}) (*sqlx.Rows, error) { - var rows *sqlx.Rows +func (d *database) Query(stmt *sqlgen.Statement, args ...interface{}) (*sqlx.Rows, error) { var query string + var p *sqlx.Stmt var err error - var start, end int64 - - start = time.Now().UnixNano() - defer func() { - end = time.Now().UnixNano() - sqlutil.Log(query, args, err, start, end) - }() + if db.Debug { + var start, end int64 + start = time.Now().UnixNano() - if d.session == nil { - return nil, db.ErrNotConnected + defer func() { + end = time.Now().UnixNano() + sqlutil.Log(query, args, err, start, end) + }() } - query = stmt.Compile(template.Template) - - if d.tx != nil { - rows, err = d.tx.Queryx(query, args...) - } else { - rows, err = d.session.Queryx(query, args...) + if p, query, err = d.prepareStatement(stmt); err != nil { + return nil, err } - return rows, err + return p.Queryx(args...) } // QueryRow compiles and executes a statement that returns at most one row. -func (d *database) QueryRow(stmt sqlgen.Statement, args ...interface{}) (*sqlx.Row, error) { +func (d *database) QueryRow(stmt *sqlgen.Statement, args ...interface{}) (*sqlx.Row, error) { var query string - var row *sqlx.Row + var p *sqlx.Stmt var err error - var start, end int64 - - start = time.Now().UnixNano() - defer func() { - end = time.Now().UnixNano() - sqlutil.Log(query, args, err, start, end) - }() + if db.Debug { + var start, end int64 + start = time.Now().UnixNano() - if d.session == nil { - return nil, db.ErrNotConnected + defer func() { + end = time.Now().UnixNano() + sqlutil.Log(query, args, err, start, end) + }() } - query = stmt.Compile(template.Template) - - if d.tx != nil { - row = d.tx.QueryRowx(query, args...) - } else { - row = d.session.QueryRowx(query, args...) + if p, query, err = d.prepareStatement(stmt); err != nil { + return nil, err } - return row, err + return p.QueryRowx(args...), nil } // populateSchema looks up for the table info in the database and populates its @@ -394,7 +427,7 @@ func (d *database) populateSchema() (err error) { d.schema = schema.NewDatabaseSchema() // Get database name. - stmt := sqlgen.Statement{ + stmt := &sqlgen.Statement{ Type: sqlgen.Select, Columns: sqlgen.JoinColumns( sqlgen.RawValue(`DATABASE()`), @@ -427,7 +460,7 @@ func (d *database) populateSchema() (err error) { } func (d *database) tableExists(names ...string) error { - var stmt sqlgen.Statement + var stmt *sqlgen.Statement var err error var rows *sqlx.Rows @@ -438,7 +471,7 @@ func (d *database) tableExists(names ...string) error { continue } - stmt = sqlgen.Statement{ + stmt = &sqlgen.Statement{ Type: sqlgen.Select, Table: sqlgen.TableWithName(`information_schema.tables`), Columns: sqlgen.JoinColumns( @@ -462,9 +495,8 @@ func (d *database) tableExists(names ...string) error { return db.ErrCollectionDoesNotExist } - defer rows.Close() - - if rows.Next() == false { + if !rows.Next() { + rows.Close() return db.ErrCollectionDoesNotExist } } @@ -481,7 +513,7 @@ func (d *database) tableColumns(tableName string) ([]string, error) { return tableSchema.Columns, nil } - stmt := sqlgen.Statement{ + stmt := &sqlgen.Statement{ Type: sqlgen.Select, Table: sqlgen.TableWithName(`information_schema.columns`), Columns: sqlgen.JoinColumns( @@ -532,7 +564,7 @@ func (d *database) getPrimaryKey(tableName string) ([]string, error) { return tableSchema.PrimaryKey, nil } - stmt := sqlgen.Statement{ + stmt := &sqlgen.Statement{ Type: sqlgen.Select, Table: sqlgen.RawValue(` information_schema.table_constraints AS t diff --git a/mysql/database_test.go b/mysql/database_test.go index 087d05651196470498f41f9cbdaa2294084041b9..fa5d9adade0a0fd82ac2228b27dd517218dcf5aa 100644 --- a/mysql/database_test.go +++ b/mysql/database_test.go @@ -600,7 +600,7 @@ func TestResultFetch(t *testing.T) { if pk, ok := rowMap["id"].(int64); !ok || pk == 0 { t.Fatalf("Expecting a not null ID.") } - if name, ok := rowMap["name"].(string); !ok || name == "" { + if name, ok := rowMap["name"].([]byte); !ok || string(name) == "" { t.Fatalf("Expecting a name.") } } else { @@ -1418,170 +1418,3 @@ func TestDataTypes(t *testing.T) { t.Fatalf("Struct is different.") } } - -// Benchmarking raw database/sql. -func BenchmarkAppendRawSQL(b *testing.B) { - var err error - var sess db.Database - - if sess, err = db.Open(Adapter, settings); err != nil { - b.Fatal(err) - } - - defer sess.Close() - - driver := sess.Driver().(*sqlx.DB) - - if _, err = driver.Exec("TRUNCATE TABLE `artist`"); err != nil { - b.Fatal(err) - } - - b.ResetTimer() - for i := 0; i < b.N; i++ { - if _, err = driver.Exec("INSERT INTO `artist` (`name`) VALUES('Hayao Miyazaki')"); err != nil { - b.Fatal(err) - } - } -} - -// Benchmarking Append(). -// -// Contributed by wei2912 -// See: https://github.com/gosexy/db/issues/20#issuecomment-20097801 -func BenchmarkAppendUpper(b *testing.B) { - sess, err := db.Open(Adapter, settings) - - if err != nil { - b.Fatal(err) - } - - defer sess.Close() - - artist, err := sess.Collection("artist") - artist.Truncate() - - item := struct { - Name string `db:"name"` - }{"Hayao Miyazaki"} - - b.ResetTimer() - for i := 0; i < b.N; i++ { - if _, err = artist.Append(item); err != nil { - b.Fatal(err) - } - } -} - -// Benchmarking raw database/sql. -func BenchmarkAppendTxRawSQL(b *testing.B) { - var err error - var sess db.Database - var tx *sql.Tx - - if sess, err = db.Open(Adapter, settings); err != nil { - b.Fatal(err) - } - - defer sess.Close() - - driver := sess.Driver().(*sqlx.DB) - - if tx, err = driver.Begin(); err != nil { - b.Fatal(err) - } - - if _, err = tx.Exec("TRUNCATE TABLE `artist`"); err != nil { - b.Fatal(err) - } - - b.ResetTimer() - for i := 0; i < b.N; i++ { - if _, err = tx.Exec("INSERT INTO `artist` (`name`) VALUES('Hayao Miyazaki')"); err != nil { - b.Fatal(err) - } - } - - if err = tx.Commit(); err != nil { - b.Fatal(err) - } -} - -// Benchmarking Append() with transactions. -func BenchmarkAppendTxUpper(b *testing.B) { - var sess db.Database - var err error - - if sess, err = db.Open(Adapter, settings); err != nil { - b.Fatal(err) - } - - defer sess.Close() - - var tx db.Tx - if tx, err = sess.Transaction(); err != nil { - b.Fatal(err) - } - - var artist db.Collection - if artist, err = tx.Collection("artist"); err != nil { - b.Fatal(err) - } - - if err = artist.Truncate(); err != nil { - b.Fatal(err) - } - - item := struct { - Name string `db:"name"` - }{"Hayao Miyazaki"} - - b.ResetTimer() - for i := 0; i < b.N; i++ { - if _, err = artist.Append(item); err != nil { - b.Fatal(err) - } - } - - if err = tx.Commit(); err != nil { - b.Fatal(err) - } -} - -// Benchmarking Append() with map. -func BenchmarkAppendTxUpperMap(b *testing.B) { - var sess db.Database - var err error - - if sess, err = db.Open(Adapter, settings); err != nil { - b.Fatal(err) - } - - defer sess.Close() - - var tx db.Tx - if tx, err = sess.Transaction(); err != nil { - b.Fatal(err) - } - - var artist db.Collection - if artist, err = tx.Collection("artist"); err != nil { - b.Fatal(err) - } - - if err = artist.Truncate(); err != nil { - b.Fatal(err) - } - - item := map[string]string{"name": "Hayao Miyazaki"} - - b.ResetTimer() - for i := 0; i < b.N; i++ { - if _, err = artist.Append(item); err != nil { - b.Fatal(err) - } - } - - if err = tx.Commit(); err != nil { - b.Fatal(err) - } -} diff --git a/postgresql/Makefile b/postgresql/Makefile index a5529f203f4a70dd63fd127f21c68079ebc87013..8f8882b0cbecbc66d9ce3ab2ef552048885dc042 100644 --- a/postgresql/Makefile +++ b/postgresql/Makefile @@ -11,4 +11,4 @@ test: reset-db $(MAKE) -C _example bench: reset-db - go test -v -test.bench=. + go test -v -test.bench=. -test.benchtime=10s -benchmem diff --git a/postgresql/_dumps/Makefile b/postgresql/_dumps/Makefile index 076eb6e051af7737be866fcbcdcf60990fbe71b1..64253b53fa959f01a287c125e97699530cb2ceda 100644 --- a/postgresql/_dumps/Makefile +++ b/postgresql/_dumps/Makefile @@ -5,10 +5,5 @@ DB_USERNAME ?= upperio_tests DB_PASSWORD ?= upperio_secret DB_NAME ?= upperio_tests -all: setup reset-db - -setup: - psql -Upostgres -h$(TEST_HOST) -p$(TEST_PORT) < setup.sql - reset-db: cat structs.sql | PGPASSWORD="$(DB_PASSWORD)" psql -U$(DB_USERNAME) $(DB_NAME) -h$(TEST_HOST) -p$(TEST_PORT) diff --git a/postgresql/benchmark_test.go b/postgresql/benchmark_test.go new file mode 100644 index 0000000000000000000000000000000000000000..7e5a6740849869bc56c89a473ccd86a9061c62f4 --- /dev/null +++ b/postgresql/benchmark_test.go @@ -0,0 +1,677 @@ +package postgresql + +import ( + "fmt" + "math/rand" + "testing" + + "github.com/jmoiron/sqlx" + "upper.io/db" +) + +const ( + testRows = 1000 +) + +func updatedArtistN(i int) string { + return fmt.Sprintf("Updated Artist %d", i%testRows) +} + +func artistN(i int) string { + return fmt.Sprintf("Artist %d", i%testRows) +} + +func connectAndAddFakeRows() (db.Database, error) { + var err error + var sess db.Database + + if sess, err = db.Open(Adapter, settings); err != nil { + return nil, err + } + + driver := sess.Driver().(*sqlx.DB) + + if _, err = driver.Exec(`TRUNCATE TABLE "artist" RESTART IDENTITY`); err != nil { + return nil, err + } + + for i := 0; i < testRows; i++ { + if _, err = driver.Exec(`INSERT INTO "artist" ("name") VALUES($1)`, artistN(i)); err != nil { + return nil, err + } + } + + return sess, nil +} + +// BenchmarkSQLAppend benchmarks raw INSERT SQL queries without using prepared +// statements nor arguments. +func BenchmarkSQLAppend(b *testing.B) { + var err error + var sess db.Database + + if sess, err = db.Open(Adapter, settings); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + if _, err = driver.Exec(`TRUNCATE TABLE "artist" RESTART IDENTITY`); err != nil { + b.Fatal(err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err = driver.Exec(`INSERT INTO "artist" ("name") VALUES('Hayao Miyazaki')`); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkSQLAppendWithArgs benchmarks raw SQL queries with arguments but +// without using prepared statements. The SQL query looks like the one that is +// generated by upper.io/db. +func BenchmarkSQLAppendWithArgs(b *testing.B) { + var err error + var sess db.Database + + if sess, err = db.Open(Adapter, settings); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + if _, err = driver.Exec(`TRUNCATE TABLE "artist" RESTART IDENTITY`); err != nil { + b.Fatal(err) + } + + args := []interface{}{ + "Hayao Miyazaki", + } + + var rows *sqlx.Rows + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if rows, err = driver.Queryx(`INSERT INTO "artist" ("name") VALUES($1) RETURNING "id"`, args...); err != nil { + b.Fatal(err) + } + rows.Close() + } +} + +// BenchmarkSQLPreparedAppend benchmarks raw INSERT SQL queries using prepared +// statements but no arguments. +func BenchmarkSQLPreparedAppend(b *testing.B) { + var err error + var sess db.Database + + if sess, err = db.Open(Adapter, settings); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + if _, err = driver.Exec(`TRUNCATE TABLE "artist" RESTART IDENTITY`); err != nil { + b.Fatal(err) + } + + stmt, err := driver.Prepare(`INSERT INTO "artist" ("name") VALUES('Hayao Miyazaki')`) + if err != nil { + b.Fatal(err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err = stmt.Exec(); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkSQLAppendWithArgs benchmarks raw INSERT SQL queries with arguments +// using prepared statements. The SQL query looks like the one that is +// generated by upper.io/db. +func BenchmarkSQLPreparedAppendWithArgs(b *testing.B) { + var err error + var sess db.Database + + if sess, err = db.Open(Adapter, settings); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + if _, err = driver.Exec(`TRUNCATE TABLE "artist"`); err != nil { + b.Fatal(err) + } + + stmt, err := driver.Preparex(`INSERT INTO "artist" ("name") VALUES($1) RETURNING "id"`) + + if err != nil { + b.Fatal(err) + } + + args := []interface{}{ + "Hayao Miyazaki", + } + + var rows *sqlx.Rows + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if rows, err = stmt.Queryx(args...); err != nil { + b.Fatal(err) + } + rows.Close() + } +} + +// BenchmarkSQLAppendWithVariableArgs benchmarks raw INSERT SQL queries with +// arguments using prepared statements. The SQL query looks like the one that +// is generated by upper.io/db. +func BenchmarkSQLPreparedAppendWithVariableArgs(b *testing.B) { + var err error + var sess db.Database + + if sess, err = db.Open(Adapter, settings); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + if _, err = driver.Exec(`TRUNCATE TABLE "artist"`); err != nil { + b.Fatal(err) + } + + stmt, err := driver.Preparex(`INSERT INTO "artist" ("name") VALUES($1) RETURNING "id"`) + + if err != nil { + b.Fatal(err) + } + + var rows *sqlx.Rows + + b.ResetTimer() + for i := 0; i < b.N; i++ { + args := []interface{}{ + fmt.Sprintf("Hayao Miyazaki %d", rand.Int()), + } + if rows, err = stmt.Queryx(args...); err != nil { + b.Fatal(err) + } + rows.Close() + } +} + +// BenchmarkSQLPreparedAppendTransactionWithArgs benchmarks raw INSERT queries +// within a transaction block with arguments and prepared statements. SQL +// queries look like those generated by upper.io/db. +func BenchmarkSQLPreparedAppendTransactionWithArgs(b *testing.B) { + var err error + var sess db.Database + var tx *sqlx.Tx + + if sess, err = db.Open(Adapter, settings); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + if tx, err = driver.Beginx(); err != nil { + b.Fatal(err) + } + + if _, err = tx.Exec(`TRUNCATE TABLE "artist" RESTART IDENTITY`); err != nil { + b.Fatal(err) + } + + stmt, err := tx.Preparex(`INSERT INTO "artist" ("name") VALUES($1) RETURNING "id"`) + if err != nil { + b.Fatal(err) + } + + args := []interface{}{ + "Hayao Miyazaki", + } + + var rows *sqlx.Rows + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if rows, err = stmt.Queryx(args...); err != nil { + b.Fatal(err) + } + rows.Close() + } + + if err = tx.Commit(); err != nil { + b.Fatal(err) + } +} + +// BenchmarkUpperAppend benchmarks an insertion by upper.io/db. +func BenchmarkUpperAppend(b *testing.B) { + + sess, err := db.Open(Adapter, settings) + if err != nil { + b.Fatal(err) + } + + defer sess.Close() + + artist, err := sess.Collection("artist") + if err != nil { + b.Fatal(err) + } + + artist.Truncate() + + item := struct { + Name string `db:"name"` + }{"Hayao Miyazaki"} + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err = artist.Append(item); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkUpperAppendVariableArgs benchmarks an insertion by upper.io/db +// with variable parameters. +func BenchmarkUpperAppendVariableArgs(b *testing.B) { + + sess, err := db.Open(Adapter, settings) + if err != nil { + b.Fatal(err) + } + + defer sess.Close() + + artist, err := sess.Collection("artist") + if err != nil { + b.Fatal(err) + } + + artist.Truncate() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + item := struct { + Name string `db:"name"` + }{fmt.Sprintf("Hayao Miyazaki %d", rand.Int())} + if _, err = artist.Append(item); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkUpperAppendTransaction benchmarks insertion queries by upper.io/db +// within a transaction operation. +func BenchmarkUpperAppendTransaction(b *testing.B) { + var sess db.Database + var err error + + if sess, err = db.Open(Adapter, settings); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + var tx db.Tx + if tx, err = sess.Transaction(); err != nil { + b.Fatal(err) + } + + var artist db.Collection + if artist, err = tx.Collection("artist"); err != nil { + b.Fatal(err) + } + + if err = artist.Truncate(); err != nil { + b.Fatal(err) + } + + item := struct { + Name string `db:"name"` + }{"Hayao Miyazaki"} + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err = artist.Append(item); err != nil { + b.Fatal(err) + } + } + + if err = tx.Commit(); err != nil { + b.Fatal(err) + } +} + +// BenchmarkUpperAppendTransactionWithMap benchmarks insertion queries by +// upper.io/db within a transaction operation using a map instead of a struct. +func BenchmarkUpperAppendTransactionWithMap(b *testing.B) { + var sess db.Database + var err error + + if sess, err = db.Open(Adapter, settings); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + var tx db.Tx + if tx, err = sess.Transaction(); err != nil { + b.Fatal(err) + } + + var artist db.Collection + if artist, err = tx.Collection("artist"); err != nil { + b.Fatal(err) + } + + if err = artist.Truncate(); err != nil { + b.Fatal(err) + } + + item := map[string]string{ + "name": "Hayao Miyazaki", + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err = artist.Append(item); err != nil { + b.Fatal(err) + } + } + + if err = tx.Commit(); err != nil { + b.Fatal(err) + } +} + +// BenchmarkSQLSelect benchmarks SQL SELECT queries. +func BenchmarkSQLSelect(b *testing.B) { + var err error + var sess db.Database + + if sess, err = connectAndAddFakeRows(); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + var res *sqlx.Rows + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if res, err = driver.Queryx(`SELECT * FROM "artist" WHERE "name" = $1`, artistN(i)); err != nil { + b.Fatal(err) + } + res.Close() + } +} + +// BenchmarkSQLPreparedSelect benchmarks SQL select queries using prepared +// statements. +func BenchmarkSQLPreparedSelect(b *testing.B) { + var err error + var sess db.Database + + if sess, err = connectAndAddFakeRows(); err != nil { + b.Fatal(err) + } + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + stmt, err := driver.Preparex(`SELECT * FROM "artist" WHERE "name" = $1`) + if err != nil { + b.Fatal(err) + } + + var res *sqlx.Rows + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if res, err = stmt.Queryx(artistN(i)); err != nil { + b.Fatal(err) + } + res.Close() + } +} + +// BenchmarkUpperFind benchmarks upper.io/db's One method. +func BenchmarkUpperFind(b *testing.B) { + var err error + var sess db.Database + + if sess, err = connectAndAddFakeRows(); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + artist, err := sess.Collection("artist") + if err != nil { + b.Fatal(err) + } + + type artistType struct { + Name string `db:"name"` + } + + var item artistType + + b.ResetTimer() + for i := 0; i < b.N; i++ { + res := artist.Find(db.Cond{"name": artistN(i)}) + if err = res.One(&item); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkUpperFindAll benchmarks upper.io/db's All method. +func BenchmarkUpperFindAll(b *testing.B) { + var err error + var sess db.Database + + if sess, err = connectAndAddFakeRows(); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + artist, err := sess.Collection("artist") + if err != nil { + b.Fatal(err) + } + + type artistType struct { + Name string `db:"name"` + } + + var items []artistType + + b.ResetTimer() + for i := 0; i < b.N; i++ { + res := artist.Find(db.Or{ + db.Cond{"name": artistN(i)}, + db.Cond{"name": artistN(i + 1)}, + db.Cond{"name": artistN(i + 2)}, + }) + if err = res.All(&items); err != nil { + b.Fatal(err) + } + if len(items) != 3 { + b.Fatal("Expecting 3 results.") + } + } +} + +// BenchmarkSQLUpdate benchmarks SQL UPDATE queries. +func BenchmarkSQLUpdate(b *testing.B) { + var err error + var sess db.Database + + if sess, err = connectAndAddFakeRows(); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err = driver.Exec(`UPDATE "artist" SET "name" = $1 WHERE "name" = $2`, updatedArtistN(i), artistN(i)); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkSQLPreparedUpdate benchmarks SQL UPDATE queries. +func BenchmarkSQLPreparedUpdate(b *testing.B) { + var err error + var sess db.Database + + if sess, err = connectAndAddFakeRows(); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + stmt, err := driver.Prepare(`UPDATE "artist" SET "name" = $1 WHERE "name" = $2`) + if err != nil { + b.Fatal(err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err = stmt.Exec(updatedArtistN(i), artistN(i)); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkUpperUpdate benchmarks upper.io/db's Update method. +func BenchmarkUpperUpdate(b *testing.B) { + var err error + var sess db.Database + + if sess, err = connectAndAddFakeRows(); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + artist, err := sess.Collection("artist") + if err != nil { + b.Fatal(err) + } + + type artistType struct { + Name string `db:"name"` + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + newValue := artistType{ + Name: updatedArtistN(i), + } + res := artist.Find(db.Cond{"name": artistN(i)}) + if err = res.Update(newValue); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkSQLDelete benchmarks SQL DELETE queries. +func BenchmarkSQLDelete(b *testing.B) { + var err error + var sess db.Database + + if sess, err = connectAndAddFakeRows(); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err = driver.Exec(`DELETE FROM "artist" WHERE "name" = $1`, artistN(i)); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkSQLPreparedDelete benchmarks SQL DELETE queries. +func BenchmarkSQLPreparedDelete(b *testing.B) { + var err error + var sess db.Database + + if sess, err = connectAndAddFakeRows(); err != nil { + b.Fatal(err) + } + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + stmt, err := driver.Prepare(`DELETE FROM "artist" WHERE "name" = $1`) + if err != nil { + b.Fatal(err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err = stmt.Exec(artistN(i)); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkUpperRemove benchmarks +func BenchmarkUpperRemove(b *testing.B) { + var err error + var sess db.Database + + if sess, err = connectAndAddFakeRows(); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + artist, err := sess.Collection("artist") + if err != nil { + b.Fatal(err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + res := artist.Find(db.Cond{"name": artistN(i)}) + if err = res.Remove(); err != nil { + b.Fatal(err) + } + } +} diff --git a/postgresql/collection.go b/postgresql/collection.go index b21ac2a48dc469f0adbf26620b5acd8685f6c0b4..2d41d9dcc9690802ccb71e5f48ecf3ff2b51ca4d 100644 --- a/postgresql/collection.go +++ b/postgresql/collection.go @@ -36,7 +36,6 @@ import ( type table struct { sqlutil.T *database - primaryKey string } var _ = db.Collection(&table{}) @@ -49,7 +48,7 @@ func (t *table) Find(terms ...interface{}) db.Result { // Truncate deletes all rows from the table. func (t *table) Truncate() error { - _, err := t.database.Exec(sqlgen.Statement{ + _, err := t.database.Exec(&sqlgen.Statement{ Type: sqlgen.Truncate, Table: sqlgen.TableWithName(t.MainTableName()), }) @@ -85,7 +84,7 @@ func (t *table) Append(item interface{}) (interface{}, error) { } } - stmt := sqlgen.Statement{ + stmt := &sqlgen.Statement{ Type: sqlgen.Insert, Table: sqlgen.TableWithName(t.MainTableName()), Columns: sqlgenCols, @@ -111,16 +110,17 @@ func (t *table) Append(item interface{}) (interface{}, error) { // A primary key was found. stmt.Extra = sqlgen.Extra(fmt.Sprintf(`RETURNING "%s"`, strings.Join(pKey, `", "`))) + if rows, err = t.database.Query(stmt, sqlgenArgs...); err != nil { return nil, err } - defer rows.Close() - keyMap := map[string]interface{}{} if err := sqlutil.FetchRow(rows, &keyMap); err != nil { + rows.Close() return nil, err } + rows.Close() // Does the item satisfy the db.IDSetter interface? if setter, ok := item.(db.IDSetter); ok { diff --git a/postgresql/database.go b/postgresql/database.go index f829e759e620549c65ac9160e5f808238494ff36..0872939404746dd009cf465b014af5b5114960dc 100644 --- a/postgresql/database.go +++ b/postgresql/database.go @@ -23,13 +23,13 @@ package postgresql import ( "database/sql" - "fmt" "strconv" "strings" "time" "github.com/jmoiron/sqlx" _ "github.com/lib/pq" // PostgreSQL driver. + "upper.io/cache" "upper.io/db" "upper.io/db/util/schema" "upper.io/db/util/sqlgen" @@ -42,10 +42,11 @@ var ( ) type database struct { - connURL db.ConnectionURL - session *sqlx.DB - tx *sqltx.Tx - schema *schema.DatabaseSchema + connURL db.ConnectionURL + session *sqlx.DB + tx *sqltx.Tx + schema *schema.DatabaseSchema + cachedStatements *cache.Cache } type tx struct { @@ -53,6 +54,11 @@ type tx struct { *database } +type cachedStatement struct { + *sqlx.Stmt + query string +} + var ( _ = db.Database(&database{}) _ = db.Tx(&tx{}) @@ -63,6 +69,52 @@ type columnSchemaT struct { DataType string `db:"data_type"` } +func (d *database) prepareStatement(stmt *sqlgen.Statement) (p *sqlx.Stmt, query string, err error) { + if d.session == nil { + return nil, "", db.ErrNotConnected + } + + pc, ok := d.cachedStatements.ReadRaw(stmt) + + if ok { + ps := pc.(*cachedStatement) + p = ps.Stmt + query = ps.query + } else { + query = compileAndReplacePlaceholders(stmt) + + if d.tx != nil { + p, err = d.tx.Preparex(query) + } else { + p, err = d.session.Preparex(query) + } + + if err != nil { + return nil, "", err + } + + d.cachedStatements.Write(stmt, &cachedStatement{p, query}) + } + + return p, query, nil +} + +func compileAndReplacePlaceholders(stmt *sqlgen.Statement) (query string) { + buf := stmt.Compile(template.Template) + + j := 1 + for i := range buf { + if buf[i] == '?' { + query = query + "$" + strconv.Itoa(j) + j++ + } else { + query = query + string(buf[i]) + } + } + + return query +} + // Driver returns the underlying *sqlx.DB instance. func (d *database) Driver() interface{} { return d.session @@ -95,8 +147,12 @@ func (d *database) Open() error { d.session.Mapper = sqlutil.NewMapper() - if err = d.populateSchema(); err != nil { - return err + d.cachedStatements = cache.NewCache() + + if d.schema == nil { + if err = d.populateSchema(); err != nil { + return err + } } return nil @@ -109,14 +165,13 @@ func (d *database) Clone() (db.Database, error) { } func (d *database) clone() (*database, error) { - src := new(database) - src.Setup(d.connURL) - - if err := src.Open(); err != nil { + clone := &database{ + schema: d.schema, + } + if err := clone.Setup(d.connURL); err != nil { return nil, err } - - return src, nil + return clone, nil } // Ping checks whether a connection to the database is still alive by pinging @@ -128,6 +183,7 @@ func (d *database) Ping() error { // Close terminates the current database session. func (d *database) Close() error { if d.session != nil { + d.cachedStatements.Clear() return d.session.Close() } return nil @@ -186,7 +242,7 @@ func (d *database) Collections() (collections []string, err error) { // Schema is empty. // Querying table names. - stmt := sqlgen.Statement{ + stmt := &sqlgen.Statement{ Type: sqlgen.Select, Columns: sqlgen.JoinColumns( sqlgen.ColumnWithName(`table_name`), @@ -207,8 +263,6 @@ func (d *database) Collections() (collections []string, err error) { return nil, err } - defer rows.Close() - collections = []string{} var name string @@ -216,6 +270,7 @@ func (d *database) Collections() (collections []string, err error) { for rows.Next() { // Getting table name. if err = rows.Scan(&name); err != nil { + rows.Close() return nil, err } @@ -230,23 +285,25 @@ func (d *database) Collections() (collections []string, err error) { } // Use changes the active database. -func (d *database) Use(database string) (err error) { +func (d *database) Use(name string) (err error) { var conn ConnectionURL if conn, err = ParseURL(d.connURL.String()); err != nil { return err } - conn.Database = database + conn.Database = name d.connURL = conn + d.schema = nil + return d.Open() } // Drop removes all tables from the current database. func (d *database) Drop() error { - _, err := d.Query(sqlgen.Statement{ + _, err := d.Query(&sqlgen.Statement{ Type: sqlgen.DropDatabase, Database: sqlgen.DatabaseWithName(d.schema.Name), }) @@ -268,8 +325,8 @@ func (d *database) Name() string { // be used to issue transactional queries. func (d *database) Transaction() (db.Tx, error) { var err error - var clone *database var sqlTx *sqlx.Tx + var clone *database if clone, err = d.clone(); err != nil { return nil, err @@ -285,105 +342,72 @@ func (d *database) Transaction() (db.Tx, error) { } // Exec compiles and executes a statement that does not return any rows. -func (d *database) Exec(stmt sqlgen.Statement, args ...interface{}) (sql.Result, error) { +func (d *database) Exec(stmt *sqlgen.Statement, args ...interface{}) (sql.Result, error) { var query string - var res sql.Result + var p *sqlx.Stmt var err error - var start, end int64 - - start = time.Now().UnixNano() - - defer func() { - end = time.Now().UnixNano() - sqlutil.Log(query, args, err, start, end) - }() - - if d.session == nil { - return nil, db.ErrNotConnected - } - query = stmt.Compile(template.Template) + if db.Debug { + var start, end int64 + start = time.Now().UnixNano() - l := len(args) - for i := 0; i < l; i++ { - query = strings.Replace(query, `?`, fmt.Sprintf(`$%d`, i+1), 1) + defer func() { + end = time.Now().UnixNano() + sqlutil.Log(query, args, err, start, end) + }() } - if d.tx != nil { - res, err = d.tx.Exec(query, args...) - } else { - res, err = d.session.Exec(query, args...) + if p, query, err = d.prepareStatement(stmt); err != nil { + return nil, err } - return res, err + return p.Exec(args...) } // Query compiles and executes a statement that returns rows. -func (d *database) Query(stmt sqlgen.Statement, args ...interface{}) (*sqlx.Rows, error) { - var rows *sqlx.Rows +func (d *database) Query(stmt *sqlgen.Statement, args ...interface{}) (*sqlx.Rows, error) { var query string + var p *sqlx.Stmt var err error - var start, end int64 - - start = time.Now().UnixNano() - - defer func() { - end = time.Now().UnixNano() - sqlutil.Log(query, args, err, start, end) - }() - if d.session == nil { - return nil, db.ErrNotConnected - } + if db.Debug { + var start, end int64 + start = time.Now().UnixNano() - query = stmt.Compile(template.Template) - - l := len(args) - for i := 0; i < l; i++ { - query = strings.Replace(query, `?`, fmt.Sprintf(`$%d`, i+1), 1) + defer func() { + end = time.Now().UnixNano() + sqlutil.Log(query, args, err, start, end) + }() } - if d.tx != nil { - rows, err = d.tx.Queryx(query, args...) - } else { - rows, err = d.session.Queryx(query, args...) + if p, query, err = d.prepareStatement(stmt); err != nil { + return nil, err } - return rows, err + return p.Queryx(args...) } // QueryRow compiles and executes a statement that returns at most one row. -func (d *database) QueryRow(stmt sqlgen.Statement, args ...interface{}) (*sqlx.Row, error) { +func (d *database) QueryRow(stmt *sqlgen.Statement, args ...interface{}) (*sqlx.Row, error) { var query string - var row *sqlx.Row + var p *sqlx.Stmt var err error - var start, end int64 - - start = time.Now().UnixNano() - - defer func() { - end = time.Now().UnixNano() - sqlutil.Log(query, args, err, start, end) - }() - - if d.session == nil { - return nil, db.ErrNotConnected - } - query = stmt.Compile(template.Template) + if db.Debug { + var start, end int64 + start = time.Now().UnixNano() - l := len(args) - for i := 0; i < l; i++ { - query = strings.Replace(query, `?`, `$`+strconv.Itoa(i+1), 1) + defer func() { + end = time.Now().UnixNano() + sqlutil.Log(query, args, err, start, end) + }() } - if d.tx != nil { - row = d.tx.QueryRowx(query, args...) - } else { - row = d.session.QueryRowx(query, args...) + if p, query, err = d.prepareStatement(stmt); err != nil { + return nil, err } - return row, err + return p.QueryRowx(args...), nil } // populateSchema looks up for the table info in the database and populates its @@ -394,7 +418,7 @@ func (d *database) populateSchema() (err error) { d.schema = schema.NewDatabaseSchema() // Get database name. - stmt := sqlgen.Statement{ + stmt := &sqlgen.Statement{ Type: sqlgen.Select, Columns: sqlgen.JoinColumns( sqlgen.RawValue(`CURRENT_DATABASE()`), @@ -425,7 +449,7 @@ func (d *database) populateSchema() (err error) { } func (d *database) tableExists(names ...string) error { - var stmt sqlgen.Statement + var stmt *sqlgen.Statement var err error var rows *sqlx.Rows @@ -436,7 +460,7 @@ func (d *database) tableExists(names ...string) error { continue } - stmt = sqlgen.Statement{ + stmt = &sqlgen.Statement{ Type: sqlgen.Select, Table: sqlgen.TableWithName(`information_schema.tables`), Columns: sqlgen.JoinColumns( @@ -460,9 +484,8 @@ func (d *database) tableExists(names ...string) error { return db.ErrCollectionDoesNotExist } - defer rows.Close() - if !rows.Next() { + rows.Close() return db.ErrCollectionDoesNotExist } } @@ -479,7 +502,7 @@ func (d *database) tableColumns(tableName string) ([]string, error) { return tableSchema.Columns, nil } - stmt := sqlgen.Statement{ + stmt := &sqlgen.Statement{ Type: sqlgen.Select, Table: sqlgen.TableWithName(`information_schema.columns`), Columns: sqlgen.JoinColumns( @@ -507,14 +530,15 @@ func (d *database) tableColumns(tableName string) ([]string, error) { return nil, err } - defer rows.Close() - tableFields := []columnSchemaT{} if err = sqlutil.FetchRows(rows, &tableFields); err != nil { + rows.Close() return nil, err } + rows.Close() + d.schema.TableInfo[tableName].Columns = make([]string, 0, len(tableFields)) for i := range tableFields { @@ -532,7 +556,7 @@ func (d *database) getPrimaryKey(tableName string) ([]string, error) { } // Getting primary key. See https://github.com/upper/db/issues/24. - stmt := sqlgen.Statement{ + stmt := &sqlgen.Statement{ Type: sqlgen.Select, Table: sqlgen.TableWithName(`pg_index, pg_class, pg_attribute`), Columns: sqlgen.JoinColumns( @@ -562,13 +586,12 @@ func (d *database) getPrimaryKey(tableName string) ([]string, error) { return nil, err } - defer rows.Close() - tableSchema.PrimaryKey = make([]string, 0, 1) for rows.Next() { var key string if err = rows.Scan(&key); err != nil { + rows.Close() return nil, err } tableSchema.PrimaryKey = append(tableSchema.PrimaryKey, key) diff --git a/postgresql/database_test.go b/postgresql/database_test.go index 3ca01716b0e6c46f6b93ea063bb4e61b1d262ac1..95a22c985e009af3d7f92f0fa2202c2181fa880a 100644 --- a/postgresql/database_test.go +++ b/postgresql/database_test.go @@ -1363,7 +1363,7 @@ func TestTransactionsAndRollback(t *testing.T) { t.Fatal(err) } - // Attempt to use the same transaction should fail. + // An attempt to use the same transaction must fail. if _, err = tx.Collection("artist"); err == nil { t.Fatalf("Illegal, transaction has already been commited.") } @@ -1549,7 +1549,6 @@ func TestCompositeKeys(t *testing.T) { // then it tries to get the stored datatypes and check if the stored and the // original values match. func TestDataTypes(t *testing.T) { - // os.Setenv(db.EnvEnableDebug, "TRUE") var res db.Result var sess db.Database @@ -1886,180 +1885,3 @@ func TestOptionTypeJsonbStruct(t *testing.T) { t.Fatalf("Expecting Num to be 123") } } - -// Benchmarking raw database/sql. -func BenchmarkAppendRawSQL(b *testing.B) { - var err error - var sess db.Database - - if sess, err = db.Open(Adapter, settings); err != nil { - b.Fatal(err) - } - - defer sess.Close() - - driver := sess.Driver().(*sqlx.DB) - - if _, err = driver.Exec(`TRUNCATE TABLE "artist"`); err != nil { - b.Fatal(err) - } - - stmt, err := driver.Prepare(`INSERT INTO "artist" ("name") VALUES('Hayao Miyazaki')`) - if err != nil { - b.Fatal(err) - } - - b.ResetTimer() - for i := 0; i < b.N; i++ { - if _, err = stmt.Exec(); err != nil { - b.Fatal(err) - } - } -} - -// Benchmarking Append(). -// -// Contributed by wei2912 -// See: https://github.com/gosexy/db/issues/20#issuecomment-20097801 -func BenchmarkAppendUpper(b *testing.B) { - sess, err := db.Open(Adapter, settings) - - if err != nil { - b.Fatal(err) - } - - defer sess.Close() - - artist, err := sess.Collection("artist") - artist.Truncate() - - item := struct { - Name string `db:"name"` - }{"Hayao Miyazaki"} - - b.ResetTimer() - for i := 0; i < b.N; i++ { - if _, err = artist.Append(item); err != nil { - b.Fatal(err) - } - } -} - -// Benchmarking raw database/sql. -func BenchmarkAppendTxRawSQL(b *testing.B) { - var err error - var sess db.Database - var tx *sql.Tx - - if sess, err = db.Open(Adapter, settings); err != nil { - b.Fatal(err) - } - - defer sess.Close() - - driver := sess.Driver().(*sqlx.DB) - - if tx, err = driver.Begin(); err != nil { - b.Fatal(err) - } - - if _, err = tx.Exec(`TRUNCATE TABLE "artist"`); err != nil { - b.Fatal(err) - } - - stmt, err := tx.Prepare(`INSERT INTO "artist" ("name") VALUES('Hayao Miyazaki')`) - if err != nil { - b.Fatal(err) - } - - b.ResetTimer() - for i := 0; i < b.N; i++ { - if _, err = stmt.Exec(); err != nil { - b.Fatal(err) - } - } - - if err = tx.Commit(); err != nil { - b.Fatal(err) - } -} - -// Benchmarking Append() with transactions. -func BenchmarkAppendTxUpper(b *testing.B) { - var sess db.Database - var err error - - if sess, err = db.Open(Adapter, settings); err != nil { - b.Fatal(err) - } - - defer sess.Close() - - var tx db.Tx - if tx, err = sess.Transaction(); err != nil { - b.Fatal(err) - } - - var artist db.Collection - if artist, err = tx.Collection("artist"); err != nil { - b.Fatal(err) - } - - if err = artist.Truncate(); err != nil { - b.Fatal(err) - } - - item := struct { - Name string `db:"name"` - }{"Hayao Miyazaki"} - - b.ResetTimer() - for i := 0; i < b.N; i++ { - if _, err = artist.Append(item); err != nil { - b.Fatal(err) - } - } - - if err = tx.Commit(); err != nil { - b.Fatal(err) - } -} - -// Benchmarking Append() with map. -func BenchmarkAppendTxUpperMap(b *testing.B) { - var sess db.Database - var err error - - if sess, err = db.Open(Adapter, settings); err != nil { - b.Fatal(err) - } - - defer sess.Close() - - var tx db.Tx - if tx, err = sess.Transaction(); err != nil { - b.Fatal(err) - } - - var artist db.Collection - if artist, err = tx.Collection("artist"); err != nil { - b.Fatal(err) - } - - if err = artist.Truncate(); err != nil { - b.Fatal(err) - } - - item := map[string]string{"name": "Hayao Miyazaki"} - - b.ResetTimer() - for i := 0; i < b.N; i++ { - if _, err = artist.Append(item); err != nil { - b.Fatal(err) - } - } - - if err = tx.Commit(); err != nil { - b.Fatal(err) - } -} diff --git a/ql/Makefile b/ql/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..527a9d7c887a2e34820b90f29bdf979ee01bd42d --- /dev/null +++ b/ql/Makefile @@ -0,0 +1,12 @@ +build: + go build && go install + +reset-db: + $(MAKE) -C _dumps + +test: reset-db + go test -v + $(MAKE) -C _example + +bench: reset-db + go test -v -test.bench=. -test.benchtime=10s -benchmem diff --git a/ql/_dumps/Makefile b/ql/_dumps/Makefile index 6828f88b8a4578ea664b52fa65fdfd0e5b43362a..8eb6708123b0f6df0024e321177e3a3f06d60137 100644 --- a/ql/_dumps/Makefile +++ b/ql/_dumps/Makefile @@ -1,5 +1,7 @@ -all: reset-db +DB_NAME ?= test.db -reset-db: - rm -f test.db - cat structs.sql | $$GOPATH/bin/ql -db test.db +load: clean + cat structs.sql | $$GOPATH/bin/ql -db $(DB_NAME) + +clean: + rm -f $(DB_NAME) diff --git a/ql/_example/Makefile b/ql/_example/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..fad39cf857334084029a0ae218f8c407c8188479 --- /dev/null +++ b/ql/_example/Makefile @@ -0,0 +1,6 @@ +DB_NAME ?= example.db + +test: + rm -f $(DB_NAME) + cat example.sql | $$GOPATH/bin/ql -db $(DB_NAME) + go run -v main.go diff --git a/ql/_example/main.go b/ql/_example/main.go index 297caef56d00518a898a1cbf1e4b1e84658e88b0..033705054afbf59f8afdc5cc62e02175991d0413 100644 --- a/ql/_example/main.go +++ b/ql/_example/main.go @@ -3,6 +3,7 @@ package main import ( "fmt" "log" + "os" "time" "upper.io/db" // Imports the main db package. @@ -16,13 +17,17 @@ var settings = db.Settings{ // Birthday struct example type Birthday struct { // Maps the "Name" property to the "name" column of the "birthdays" table. - Name string `field:"name"` + Name string `db:"name"` // Maps the "Born" property to the "born" column of the "birthdays" table. - Born time.Time `field:"born"` + Born time.Time `db:"born"` } func main() { + if os.Getenv("DB_NAME") != "" { + settings.Database = os.Getenv("DB_NAME") + } + // Attemping to open the "example.db" database file. sess, err := db.Open("ql", settings) diff --git a/ql/benchmark_test.go b/ql/benchmark_test.go new file mode 100644 index 0000000000000000000000000000000000000000..32e52d029a497adbb96076ce7b850347c23c5b0a --- /dev/null +++ b/ql/benchmark_test.go @@ -0,0 +1,789 @@ +package ql + +import ( + "fmt" + "math/rand" + "testing" + + "github.com/jmoiron/sqlx" + "upper.io/db" +) + +const ( + testRows = 1000 +) + +func updatedArtistN(i int) string { + return fmt.Sprintf("Updated Artist %d", i%testRows) +} + +func artistN(i int) string { + return fmt.Sprintf("Artist %d", i%testRows) +} + +func connectAndAddFakeRows() (db.Database, error) { + var err error + var sess db.Database + var tx *sqlx.Tx + + if sess, err = db.Open(Adapter, settings); err != nil { + return nil, err + } + + driver := sess.Driver().(*sqlx.DB) + + if tx, err = driver.Beginx(); err != nil { + return nil, err + } + + if _, err = tx.Exec(`TRUNCATE TABLE artist`); err != nil { + return nil, err + } + + stmt, err := tx.Preparex(`INSERT INTO artist (name) VALUES($1)`) + if err != nil { + return nil, err + } + + for i := 0; i < testRows; i++ { + if _, err = stmt.Exec(artistN(i)); err != nil { + return nil, err + } + } + + if err = tx.Commit(); err != nil { + return nil, err + } + + return sess, nil +} + +// BenchmarkSQLAppend benchmarks raw INSERT SQL queries without using prepared +// statements nor arguments. +func BenchmarkSQLAppend(b *testing.B) { + var err error + var sess db.Database + var tx *sqlx.Tx + + if sess, err = db.Open(Adapter, settings); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + if tx, err = driver.Beginx(); err != nil { + b.Fatal(err) + } + if _, err = tx.Exec(`TRUNCATE TABLE artist`); err != nil { + b.Fatal(err) + } + if err = tx.Commit(); err != nil { + b.Fatal(err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if tx, err = driver.Beginx(); err != nil { + b.Fatal(err) + } + if _, err = tx.Exec(`INSERT INTO artist (name) VALUES("Hayao Miyazaki")`); err != nil { + b.Fatal(err) + } + if err = tx.Commit(); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkSQLAppendWithArgs benchmarks raw SQL queries with arguments but +// without using prepared statements. The SQL query looks like the one that is +// generated by upper.io/db. +func BenchmarkSQLAppendWithArgs(b *testing.B) { + var err error + var sess db.Database + var tx *sqlx.Tx + + if sess, err = db.Open(Adapter, settings); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + if tx, err = driver.Beginx(); err != nil { + b.Fatal(err) + } + if _, err = tx.Exec(`TRUNCATE TABLE artist`); err != nil { + b.Fatal(err) + } + if err = tx.Commit(); err != nil { + b.Fatal(err) + } + + args := []interface{}{ + "Hayao Miyazaki", + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if tx, err = driver.Beginx(); err != nil { + b.Fatal(err) + } + if _, err = tx.Exec(`INSERT INTO artist (name) VALUES($1)`, args...); err != nil { + b.Fatal(err) + } + if err = tx.Commit(); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkSQLPreparedAppend benchmarks raw INSERT SQL queries using prepared +// statements but no arguments. +func BenchmarkSQLPreparedAppend(b *testing.B) { + var err error + var sess db.Database + var tx *sqlx.Tx + + if sess, err = db.Open(Adapter, settings); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + if tx, err = driver.Beginx(); err != nil { + b.Fatal(err) + } + if _, err = tx.Exec(`TRUNCATE TABLE artist`); err != nil { + b.Fatal(err) + } + if err = tx.Commit(); err != nil { + b.Fatal(err) + } + + stmt, err := driver.Prepare(`INSERT INTO artist (name) VALUES("Hayao Miyazaki")`) + if err != nil { + b.Fatal(err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if tx, err = driver.Beginx(); err != nil { + b.Fatal(err) + } + if _, err = tx.Stmtx(stmt).Exec(); err != nil { + b.Fatal(err) + } + if err = tx.Commit(); err != nil { + b.Fatal(err) + } + } + +} + +// BenchmarkSQLAppendWithArgs benchmarks raw INSERT SQL queries with arguments +// using prepared statements. The SQL query looks like the one that is +// generated by upper.io/db. +func BenchmarkSQLPreparedAppendWithArgs(b *testing.B) { + var err error + var sess db.Database + var tx *sqlx.Tx + + if sess, err = db.Open(Adapter, settings); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + if tx, err = driver.Beginx(); err != nil { + b.Fatal(err) + } + if _, err = tx.Exec(`TRUNCATE TABLE artist`); err != nil { + b.Fatal(err) + } + if err = tx.Commit(); err != nil { + b.Fatal(err) + } + + stmt, err := driver.Prepare(`INSERT INTO artist (name) VALUES($1)`) + if err != nil { + b.Fatal(err) + } + + args := []interface{}{ + "Hayao Miyazaki", + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if tx, err = driver.Beginx(); err != nil { + b.Fatal(err) + } + if _, err = tx.Stmtx(stmt).Exec(args...); err != nil { + b.Fatal(err) + } + if err = tx.Commit(); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkSQLAppendWithVariableArgs benchmarks raw INSERT SQL queries with +// arguments using prepared statements. The SQL query looks like the one that +// is generated by upper.io/db. +func BenchmarkSQLPreparedAppendWithVariableArgs(b *testing.B) { + var err error + var sess db.Database + var tx *sqlx.Tx + + if sess, err = db.Open(Adapter, settings); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + if tx, err = driver.Beginx(); err != nil { + b.Fatal(err) + } + if _, err = tx.Exec(`TRUNCATE TABLE artist`); err != nil { + b.Fatal(err) + } + if err = tx.Commit(); err != nil { + b.Fatal(err) + } + + stmt, err := driver.Prepare(`INSERT INTO artist (name) VALUES($1)`) + if err != nil { + b.Fatal(err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + args := []interface{}{ + fmt.Sprintf("Hayao Miyazaki %d", rand.Int()), + } + if tx, err = driver.Beginx(); err != nil { + b.Fatal(err) + } + if _, err = tx.Stmtx(stmt).Exec(args...); err != nil { + b.Fatal(err) + } + if err = tx.Commit(); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkSQLPreparedAppendTransactionWithArgs benchmarks raw INSERT queries +// within a transaction block with arguments and prepared statements. SQL +// queries look like those generated by upper.io/db. +func BenchmarkSQLPreparedAppendTransactionWithArgs(b *testing.B) { + var err error + var sess db.Database + var tx *sqlx.Tx + + if sess, err = db.Open(Adapter, settings); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + if tx, err = driver.Beginx(); err != nil { + b.Fatal(err) + } + + if _, err = tx.Exec("TRUNCATE TABLE artist"); err != nil { + b.Fatal(err) + } + + stmt, err := tx.Preparex(`INSERT INTO artist (name) VALUES($1)`) + if err != nil { + b.Fatal(err) + } + + args := []interface{}{ + "Hayao Miyazaki", + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err = stmt.Exec(args...); err != nil { + b.Fatal(err) + } + } + + if err = tx.Commit(); err != nil { + b.Fatal(err) + } +} + +// BenchmarkUpperAppend benchmarks an insertion by upper.io/db. +func BenchmarkUpperAppend(b *testing.B) { + + sess, err := db.Open(Adapter, settings) + if err != nil { + b.Fatal(err) + } + + defer sess.Close() + + artist, err := sess.Collection("artist") + if err != nil { + b.Fatal(err) + } + + artist.Truncate() + + item := struct { + Name string `db:"name"` + }{"Hayao Miyazaki"} + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err = artist.Append(item); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkUpperAppendVariableArgs benchmarks an insertion by upper.io/db +// with variable parameters. +func BenchmarkUpperAppendVariableArgs(b *testing.B) { + + sess, err := db.Open(Adapter, settings) + if err != nil { + b.Fatal(err) + } + + defer sess.Close() + + artist, err := sess.Collection("artist") + if err != nil { + b.Fatal(err) + } + + artist.Truncate() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + item := struct { + Name string `db:"name"` + }{fmt.Sprintf("Hayao Miyazaki %d", rand.Int())} + if _, err = artist.Append(item); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkUpperAppendTransaction benchmarks insertion queries by upper.io/db +// within a transaction operation. +func BenchmarkUpperAppendTransaction(b *testing.B) { + var sess db.Database + var err error + + if sess, err = db.Open(Adapter, settings); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + var tx db.Tx + if tx, err = sess.Transaction(); err != nil { + b.Fatal(err) + } + + var artist db.Collection + if artist, err = tx.Collection("artist"); err != nil { + b.Fatal(err) + } + + if err = artist.Truncate(); err != nil { + b.Fatal(err) + } + + item := struct { + Name string `db:"name"` + }{"Hayao Miyazaki"} + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err = artist.Append(item); err != nil { + b.Fatal(err) + } + } + + if err = tx.Commit(); err != nil { + b.Fatal(err) + } +} + +// BenchmarkUpperAppendTransactionWithMap benchmarks insertion queries by +// upper.io/db within a transaction operation using a map instead of a struct. +func BenchmarkUpperAppendTransactionWithMap(b *testing.B) { + var sess db.Database + var err error + + if sess, err = db.Open(Adapter, settings); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + var tx db.Tx + if tx, err = sess.Transaction(); err != nil { + b.Fatal(err) + } + + var artist db.Collection + if artist, err = tx.Collection("artist"); err != nil { + b.Fatal(err) + } + + if err = artist.Truncate(); err != nil { + b.Fatal(err) + } + + item := map[string]string{ + "name": "Hayao Miyazaki", + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err = artist.Append(item); err != nil { + b.Fatal(err) + } + } + + if err = tx.Commit(); err != nil { + b.Fatal(err) + } +} + +// BenchmarkSQLSelect benchmarks SQL SELECT queries. +func BenchmarkSQLSelect(b *testing.B) { + var err error + var sess db.Database + + connectAndAddFakeRows() + /* + // This is failing for some reason, I suspect that QL's Close() removes + // some cached value that the next Open() tries to use. + if sess, err = connectAndAddFakeRows(); err != nil { + b.Fatal(err) + } + */ + + if sess, err = db.Open(Adapter, settings); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + var rows *sqlx.Rows + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if rows, err = driver.Queryx(`SELECT * FROM artist WHERE name == $1`, artistN(i)); err != nil { + b.Fatal(err) + } + rows.Close() + } +} + +// BenchmarkSQLPreparedSelect benchmarks SQL select queries using prepared +// statements. +func BenchmarkSQLPreparedSelect(b *testing.B) { + var err error + var sess db.Database + + connectAndAddFakeRows() + /* + // This is failing for some reason, I suspect that QL's Close() removes + // some cached value that the next Open() tries to use. + if sess, err = connectAndAddFakeRows(); err != nil { + b.Fatal(err) + } + */ + + if sess, err = db.Open(Adapter, settings); err != nil { + b.Fatal(err) + } + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + stmt, err := driver.Preparex(`SELECT * FROM artist WHERE name == $1`) + if err != nil { + b.Fatal(err) + } + + var rows *sqlx.Rows + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if rows, err = stmt.Queryx(artistN(i)); err != nil { + b.Fatal(err) + } + rows.Close() + } +} + +// BenchmarkUpperFind benchmarks upper.io/db's One method. +func BenchmarkUpperFind(b *testing.B) { + var err error + var sess db.Database + + if sess, err = connectAndAddFakeRows(); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + artist, err := sess.Collection("artist") + if err != nil { + b.Fatal(err) + } + + type artistType struct { + Name string `db:"name"` + } + + var item artistType + + b.ResetTimer() + for i := 0; i < b.N; i++ { + res := artist.Find(db.Cond{"name": artistN(i)}) + if err = res.One(&item); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkUpperFindAll benchmarks upper.io/db's All method. +func BenchmarkUpperFindAll(b *testing.B) { + var err error + var sess db.Database + + if sess, err = connectAndAddFakeRows(); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + artist, err := sess.Collection("artist") + if err != nil { + b.Fatal(err) + } + + type artistType struct { + Name string `db:"name"` + } + + var items []artistType + + b.ResetTimer() + for i := 0; i < b.N; i++ { + res := artist.Find(db.Or{ + db.Cond{"name": artistN(i)}, + db.Cond{"name": artistN(i + 1)}, + db.Cond{"name": artistN(i + 2)}, + }) + if err = res.All(&items); err != nil { + b.Fatal(err) + } + if len(items) != 3 { + b.Fatal("Expecting 3 results.") + } + } +} + +// BenchmarkSQLUpdate benchmarks SQL UPDATE queries. +func BenchmarkSQLUpdate(b *testing.B) { + var err error + var sess db.Database + var tx *sqlx.Tx + + if sess, err = connectAndAddFakeRows(); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if tx, err = driver.Beginx(); err != nil { + b.Fatal(err) + } + if _, err = tx.Exec(`UPDATE artist SET name = $1 WHERE name == $2`, updatedArtistN(i), artistN(i)); err != nil { + b.Fatal(err) + } + if err = tx.Commit(); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkSQLPreparedUpdate benchmarks SQL UPDATE queries. +func BenchmarkSQLPreparedUpdate(b *testing.B) { + var err error + var sess db.Database + var tx *sqlx.Tx + + if sess, err = connectAndAddFakeRows(); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + stmt, err := driver.Prepare(`UPDATE artist SET name = $1 WHERE name == $2`) + if err != nil { + b.Fatal(err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if tx, err = driver.Beginx(); err != nil { + b.Fatal(err) + } + if _, err = tx.Stmtx(stmt).Exec(updatedArtistN(i), artistN(i)); err != nil { + b.Fatal(err) + } + if err = tx.Commit(); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkUpperUpdate benchmarks upper.io/db's Update method. +func BenchmarkUpperUpdate(b *testing.B) { + var err error + var sess db.Database + + if sess, err = connectAndAddFakeRows(); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + artist, err := sess.Collection("artist") + if err != nil { + b.Fatal(err) + } + + type artistType struct { + Name string `db:"name"` + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + newValue := artistType{ + Name: updatedArtistN(i), + } + res := artist.Find(db.Cond{"name": artistN(i)}) + if err = res.Update(newValue); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkSQLDelete benchmarks SQL DELETE queries. +func BenchmarkSQLDelete(b *testing.B) { + var err error + var sess db.Database + var tx *sqlx.Tx + + if sess, err = connectAndAddFakeRows(); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if tx, err = driver.Beginx(); err != nil { + b.Fatal(err) + } + if _, err = tx.Exec(`DELETE FROM artist WHERE name == $1`, artistN(i)); err != nil { + b.Fatal(err) + } + if err = tx.Commit(); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkSQLPreparedDelete benchmarks SQL DELETE queries. +func BenchmarkSQLPreparedDelete(b *testing.B) { + var err error + var sess db.Database + var tx *sqlx.Tx + + if sess, err = connectAndAddFakeRows(); err != nil { + b.Fatal(err) + } + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + stmt, err := driver.Prepare(`DELETE FROM artist WHERE name == $1`) + if err != nil { + b.Fatal(err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if tx, err = driver.Beginx(); err != nil { + b.Fatal(err) + } + if _, err = tx.Stmtx(stmt).Exec(artistN(i)); err != nil { + b.Fatal(err) + } + if err = tx.Commit(); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkUpperRemove benchmarks +func BenchmarkUpperRemove(b *testing.B) { + var err error + var sess db.Database + + if sess, err = connectAndAddFakeRows(); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + artist, err := sess.Collection("artist") + if err != nil { + b.Fatal(err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + res := artist.Find(db.Cond{"name": artistN(i)}) + if err = res.Remove(); err != nil { + b.Fatal(err) + } + } +} diff --git a/ql/collection.go b/ql/collection.go index 110aec9166e81c8912673c49ec68fb7cf93bbd58..b6b5fcf3007a58cb5a9cdc418c7782cf9defef98 100644 --- a/ql/collection.go +++ b/ql/collection.go @@ -49,7 +49,7 @@ func (t *table) Find(terms ...interface{}) db.Result { // Truncate deletes all rows from the table. func (t *table) Truncate() error { - _, err := t.database.Exec(sqlgen.Statement{ + _, err := t.database.Exec(&sqlgen.Statement{ Type: sqlgen.Truncate, Table: sqlgen.TableWithName(t.MainTableName()), }) @@ -75,7 +75,7 @@ func (t *table) Append(item interface{}) (interface{}, error) { return nil, err } - stmt := sqlgen.Statement{ + stmt := &sqlgen.Statement{ Type: sqlgen.Insert, Table: sqlgen.TableWithName(t.MainTableName()), Columns: sqlgenCols, diff --git a/ql/database.go b/ql/database.go index f06851052aefd0ca35f81533d723ded7e2612856..949d79386b65c0aa250fd2ae49b90e9b46267140 100644 --- a/ql/database.go +++ b/ql/database.go @@ -23,12 +23,13 @@ package ql import ( "database/sql" - "fmt" + "strconv" "strings" "time" _ "github.com/cznic/ql/driver" // QL driver "github.com/jmoiron/sqlx" + "upper.io/cache" "upper.io/db" "upper.io/db/util/schema" "upper.io/db/util/sqlgen" @@ -41,10 +42,11 @@ var ( ) type database struct { - connURL db.ConnectionURL - session *sqlx.DB - tx *sqltx.Tx - schema *schema.DatabaseSchema + connURL db.ConnectionURL + session *sqlx.DB + tx *sqltx.Tx + schema *schema.DatabaseSchema + cachedStatements *cache.Cache } type tx struct { @@ -52,6 +54,11 @@ type tx struct { *database } +type cachedStatement struct { + *sqlx.Stmt + query string +} + var ( _ = db.Database(&database{}) _ = db.Tx(&tx{}) @@ -61,6 +68,52 @@ type columnSchemaT struct { Name string `db:"Name"` } +func (d *database) prepareStatement(stmt *sqlgen.Statement) (p *sqlx.Stmt, query string, err error) { + if d.session == nil { + return nil, "", db.ErrNotConnected + } + + pc, ok := d.cachedStatements.ReadRaw(stmt) + + if ok { + ps := pc.(*cachedStatement) + p = ps.Stmt + query = ps.query + } else { + query = compileAndReplacePlaceholders(stmt) + + if d.tx != nil { + p, err = d.tx.Preparex(query) + } else { + p, err = d.session.Preparex(query) + } + + if err != nil { + return nil, "", err + } + + d.cachedStatements.Write(stmt, &cachedStatement{p, query}) + } + + return p, query, nil +} + +func compileAndReplacePlaceholders(stmt *sqlgen.Statement) (query string) { + buf := stmt.Compile(template.Template) + + j := 1 + for i := range buf { + if buf[i] == '?' { + query = query + "$" + strconv.Itoa(j) + j++ + } else { + query = query + string(buf[i]) + } + } + + return query +} + // Driver returns the underlying *sqlx.DB instance. func (d *database) Driver() interface{} { return d.session @@ -89,8 +142,12 @@ func (d *database) Open() error { d.session.Mapper = sqlutil.NewMapper() - if err = d.populateSchema(); err != nil { - return err + d.cachedStatements = cache.NewCache() + + if d.schema == nil { + if err = d.populateSchema(); err != nil { + return err + } } return nil @@ -103,13 +160,13 @@ func (d *database) Clone() (db.Database, error) { } func (d *database) clone() (adapter *database, err error) { - adapter = new(database) - - if err = adapter.Setup(d.connURL); err != nil { + clone := &database{ + schema: d.schema, + } + if err := clone.Setup(d.connURL); err != nil { return nil, err } - - return adapter, nil + return clone, nil } // Ping checks whether a connection to the database is still alive by pinging @@ -179,7 +236,7 @@ func (d *database) Collections() (collections []string, err error) { // Schema is empty. // Querying table names. - stmt := sqlgen.Statement{ + stmt := &sqlgen.Statement{ Type: sqlgen.Select, Table: sqlgen.TableWithName(`__Table`), Columns: sqlgen.JoinColumns( @@ -201,6 +258,7 @@ func (d *database) Collections() (collections []string, err error) { for rows.Next() { // Getting table name. + if err = rows.Scan(&name); err != nil { return nil, err } @@ -216,17 +274,19 @@ func (d *database) Collections() (collections []string, err error) { } // Use changes the active database. -func (d *database) Use(database string) (err error) { +func (d *database) Use(name string) (err error) { var conn ConnectionURL if conn, err = ParseURL(d.connURL.String()); err != nil { return err } - conn.Database = database + conn.Database = name d.connURL = conn + d.schema = nil + return d.Open() } @@ -267,141 +327,93 @@ func (d *database) Transaction() (db.Tx, error) { } // Exec compiles and executes a statement that does not return any rows. -func (d *database) Exec(stmt sqlgen.Statement, args ...interface{}) (sql.Result, error) { +func (d *database) Exec(stmt *sqlgen.Statement, args ...interface{}) (sql.Result, error) { var query string - var res sql.Result + var p *sqlx.Stmt var err error - var start, end int64 - - start = time.Now().UnixNano() - defer func() { - end = time.Now().UnixNano() - sqlutil.Log(query, args, err, start, end) - }() + if db.Debug { + var start, end int64 + start = time.Now().UnixNano() - if d.session == nil { - return nil, db.ErrNotConnected + defer func() { + end = time.Now().UnixNano() + sqlutil.Log(query, args, err, start, end) + }() } - query = stmt.Compile(template.Template) - - l := len(args) - for i := 0; i < l; i++ { - query = strings.Replace(query, `?`, fmt.Sprintf(`$%d`, i+1), 1) + if p, query, err = d.prepareStatement(stmt); err != nil { + return nil, err } - if d.tx != nil { - res, err = d.tx.Exec(query, args...) - } else { + if d.tx == nil { var tx *sqlx.Tx + var res sql.Result if tx, err = d.session.Beginx(); err != nil { return nil, err } - if res, err = tx.Exec(query, args...); err != nil { + s := tx.Stmtx(p) + + if res, err = s.Exec(args...); err != nil { return nil, err } if err = tx.Commit(); err != nil { return nil, err } + + return res, err } - return res, err + return p.Exec(args...) } // Query compiles and executes a statement that returns rows. -func (d *database) Query(stmt sqlgen.Statement, args ...interface{}) (*sqlx.Rows, error) { - var rows *sqlx.Rows +func (d *database) Query(stmt *sqlgen.Statement, args ...interface{}) (*sqlx.Rows, error) { var query string + var p *sqlx.Stmt var err error - var start, end int64 - start = time.Now().UnixNano() + if db.Debug { + var start, end int64 + start = time.Now().UnixNano() - defer func() { - end = time.Now().UnixNano() - sqlutil.Log(query, args, err, start, end) - }() - - if d.session == nil { - return nil, db.ErrNotConnected - } - - query = stmt.Compile(template.Template) - - l := len(args) - for i := 0; i < l; i++ { - query = strings.Replace(query, `?`, fmt.Sprintf(`$%d`, i+1), 1) + defer func() { + end = time.Now().UnixNano() + sqlutil.Log(query, args, err, start, end) + }() } - if d.tx != nil { - rows, err = d.tx.Queryx(query, args...) - } else { - var tx *sqlx.Tx - - if tx, err = d.session.Beginx(); err != nil { - return nil, err - } - - if rows, err = tx.Queryx(query, args...); err != nil { - return nil, err - } - - if err = tx.Commit(); err != nil { - return nil, err - } + if p, query, err = d.prepareStatement(stmt); err != nil { + return nil, err } - return rows, err + return p.Queryx(args...) } // QueryRow compiles and executes a statement that returns at most one row. -func (d *database) QueryRow(stmt sqlgen.Statement, args ...interface{}) (*sqlx.Row, error) { +func (d *database) QueryRow(stmt *sqlgen.Statement, args ...interface{}) (*sqlx.Row, error) { var query string - var row *sqlx.Row + var p *sqlx.Stmt var err error - var start, end int64 - start = time.Now().UnixNano() + if db.Debug { + var start, end int64 + start = time.Now().UnixNano() - defer func() { - end = time.Now().UnixNano() - sqlutil.Log(query, args, err, start, end) - }() - - if d.session == nil { - return nil, db.ErrNotConnected - } - - query = stmt.Compile(template.Template) - - l := len(args) - for i := 0; i < l; i++ { - query = strings.Replace(query, `?`, fmt.Sprintf(`$%d`, i+1), 1) + defer func() { + end = time.Now().UnixNano() + sqlutil.Log(query, args, err, start, end) + }() } - if d.tx != nil { - row = d.tx.QueryRowx(query, args...) - } else { - var tx *sqlx.Tx - - if tx, err = d.session.Beginx(); err != nil { - return nil, err - } - - if row = tx.QueryRowx(query, args...); err != nil { - return nil, err - } - - if err = tx.Commit(); err != nil { - return nil, err - } + if p, query, err = d.prepareStatement(stmt); err != nil { + return nil, err } - return row, err + return p.QueryRowx(args...), nil } // populateSchema looks up for the table info in the database and populates its @@ -435,7 +447,7 @@ func (d *database) populateSchema() (err error) { } func (d *database) tableExists(names ...string) error { - var stmt sqlgen.Statement + var stmt *sqlgen.Statement var err error var rows *sqlx.Rows @@ -446,7 +458,7 @@ func (d *database) tableExists(names ...string) error { continue } - stmt = sqlgen.Statement{ + stmt = &sqlgen.Statement{ Type: sqlgen.Select, Table: sqlgen.TableWithName(`__Table`), Columns: sqlgen.JoinColumns( @@ -484,7 +496,7 @@ func (d *database) tableColumns(tableName string) ([]string, error) { return tableSchema.Columns, nil } - stmt := sqlgen.Statement{ + stmt := &sqlgen.Statement{ Type: sqlgen.Select, Table: sqlgen.TableWithName(`__Column`), Columns: sqlgen.JoinColumns( diff --git a/ql/database_test.go b/ql/database_test.go index 4200d28dbfa71804261bb53d2ead7116d4a1c755..057a5ba896e5d849bc938feb44dc8062fd812e71 100644 --- a/ql/database_test.go +++ b/ql/database_test.go @@ -29,7 +29,7 @@ package ql // go test import ( - "database/sql" + "os" //"reflect" //"errors" @@ -1241,190 +1241,3 @@ func TestDataTypes(t *testing.T) { } } */ - -// Benchmarking raw database/sql. -func BenchmarkAppendRawSQL(b *testing.B) { - var err error - var sess db.Database - var tx *sql.Tx - - if sess, err = db.Open(Adapter, settings); err != nil { - b.Fatal(err) - } - - defer sess.Close() - - driver := sess.Driver().(*sqlx.DB) - - if tx, err = driver.Begin(); err != nil { - b.Fatal(err) - } - - if _, err = tx.Exec(`TRUNCATE TABLE artist`); err != nil { - b.Fatal(err) - } - - if err = tx.Commit(); err != nil { - b.Fatal(err) - } - - b.ResetTimer() - for i := 0; i < b.N; i++ { - if tx, err = driver.Begin(); err != nil { - b.Fatal(err) - } - if _, err = tx.Exec(`INSERT INTO artist (name) VALUES("Hayao Miyazaki")`); err != nil { - b.Fatal(err) - } - if err = tx.Commit(); err != nil { - b.Fatal(err) - } - } -} - -// Benchmarking Append(). -// -// Contributed by wei2912 -// See: https://github.com/gosexy/db/issues/20#issuecomment-20097801 -func BenchmarkAppendUpper(b *testing.B) { - var sess db.Database - var artist db.Collection - var err error - - if sess, err = db.Open(Adapter, settings); err != nil { - b.Fatal(err) - } - - defer sess.Close() - - if artist, err = sess.Collection("artist"); err != nil { - b.Fatal(err) - } - - artist.Truncate() - - item := struct { - Name string `db:"name"` - }{"Hayao Miyazaki"} - - b.ResetTimer() - for i := 0; i < b.N; i++ { - if _, err = artist.Append(item); err != nil { - b.Fatal(err) - } - } -} - -// Benchmarking raw database/sql. -func BenchmarkAppendTxRawSQL(b *testing.B) { - var err error - var sess db.Database - var tx *sql.Tx - - if sess, err = db.Open(Adapter, settings); err != nil { - b.Fatal(err) - } - - defer sess.Close() - - driver := sess.Driver().(*sqlx.DB) - - if tx, err = driver.Begin(); err != nil { - b.Fatal(err) - } - - if _, err = tx.Exec(`TRUNCATE TABLE artist`); err != nil { - b.Fatal(err) - } - - b.ResetTimer() - for i := 0; i < b.N; i++ { - if _, err = tx.Exec(`INSERT INTO artist (name) VALUES("Hayao Miyazaki")`); err != nil { - b.Fatal(err) - } - } - - if err = tx.Commit(); err != nil { - b.Fatal(err) - } -} - -// Benchmarking Append() with transactions. -func BenchmarkAppendTxUpper(b *testing.B) { - var sess db.Database - var err error - var tx db.Tx - - if sess, err = db.Open(Adapter, settings); err != nil { - b.Fatal(err) - } - - defer sess.Close() - - if tx, err = sess.Transaction(); err != nil { - b.Fatal(err) - } - - var artist db.Collection - if artist, err = tx.Collection("artist"); err != nil { - b.Fatal(err) - } - - if err = artist.Truncate(); err != nil { - b.Fatal(err) - } - - item := struct { - Name string `db:"name"` - }{"Hayao Miyazaki"} - - b.ResetTimer() - for i := 0; i < b.N; i++ { - if _, err = artist.Append(item); err != nil { - b.Fatal(err) - } - } - - if err = tx.Commit(); err != nil { - b.Fatal(err) - } -} - -// Benchmarking Append() with map. -func BenchmarkAppendTxUpperMap(b *testing.B) { - var sess db.Database - var err error - - if sess, err = db.Open(Adapter, settings); err != nil { - b.Fatal(err) - } - - defer sess.Close() - - var tx db.Tx - if tx, err = sess.Transaction(); err != nil { - b.Fatal(err) - } - - var artist db.Collection - if artist, err = tx.Collection("artist"); err != nil { - b.Fatal(err) - } - - if err = artist.Truncate(); err != nil { - b.Fatal(err) - } - - item := map[string]string{"name": "Hayao Miyazaki"} - - b.ResetTimer() - for i := 0; i < b.N; i++ { - if _, err = artist.Append(item); err != nil { - b.Fatal(err) - } - } - - if err = tx.Commit(); err != nil { - b.Fatal(err) - } -} diff --git a/sqlite/Makefile b/sqlite/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..527a9d7c887a2e34820b90f29bdf979ee01bd42d --- /dev/null +++ b/sqlite/Makefile @@ -0,0 +1,12 @@ +build: + go build && go install + +reset-db: + $(MAKE) -C _dumps + +test: reset-db + go test -v + $(MAKE) -C _example + +bench: reset-db + go test -v -test.bench=. -test.benchtime=10s -benchmem diff --git a/sqlite/_example/Makefile b/sqlite/_example/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..04d4100ddd2a31ae26a67e97e4768cbb01699edf --- /dev/null +++ b/sqlite/_example/Makefile @@ -0,0 +1,4 @@ +test: + rm -f example.db + cat example.sql | sqlite3 example.db + go run -v main.go diff --git a/sqlite/benchmark_test.go b/sqlite/benchmark_test.go new file mode 100644 index 0000000000000000000000000000000000000000..27447312385e44d0bbbcf474608c76aa19728d3c --- /dev/null +++ b/sqlite/benchmark_test.go @@ -0,0 +1,665 @@ +package sqlite + +import ( + "fmt" + "math/rand" + "testing" + + "github.com/jmoiron/sqlx" + "upper.io/db" +) + +const ( + testRows = 1000 +) + +func updatedArtistN(i int) string { + return fmt.Sprintf("Updated Artist %d", i%testRows) +} + +func artistN(i int) string { + return fmt.Sprintf("Artist %d", i%testRows) +} + +func connectAndAddFakeRows() (db.Database, error) { + var err error + var sess db.Database + + if sess, err = db.Open(Adapter, settings); err != nil { + return nil, err + } + + driver := sess.Driver().(*sqlx.DB) + + if _, err = driver.Exec(`DELETE FROM "artist"`); err != nil { + return nil, err + } + + for i := 0; i < testRows; i++ { + if _, err = driver.Exec(`INSERT INTO "artist" ("name") VALUES(?)`, artistN(i)); err != nil { + return nil, err + } + } + + return sess, nil +} + +// BenchmarkSQLAppend benchmarks raw INSERT SQL queries without using prepared +// statements nor arguments. +func BenchmarkSQLAppend(b *testing.B) { + var err error + var sess db.Database + + if sess, err = db.Open(Adapter, settings); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + if _, err = driver.Exec(`DELETE FROM "artist"`); err != nil { + b.Fatal(err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err = driver.Exec(`INSERT INTO "artist" ("name") VALUES('Hayao Miyazaki')`); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkSQLAppendWithArgs benchmarks raw SQL queries with arguments but +// without using prepared statements. The SQL query looks like the one that is +// generated by upper.io/db. +func BenchmarkSQLAppendWithArgs(b *testing.B) { + var err error + var sess db.Database + + if sess, err = db.Open(Adapter, settings); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + if _, err = driver.Exec(`DELETE FROM "artist"`); err != nil { + b.Fatal(err) + } + + args := []interface{}{ + "Hayao Miyazaki", + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err = driver.Exec(`INSERT INTO "artist" ("name") VALUES(?)`, args...); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkSQLPreparedAppend benchmarks raw INSERT SQL queries using prepared +// statements but no arguments. +func BenchmarkSQLPreparedAppend(b *testing.B) { + var err error + var sess db.Database + + if sess, err = db.Open(Adapter, settings); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + if _, err = driver.Exec(`DELETE FROM "artist"`); err != nil { + b.Fatal(err) + } + + stmt, err := driver.Prepare(`INSERT INTO "artist" ("name") VALUES('Hayao Miyazaki')`) + if err != nil { + b.Fatal(err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err = stmt.Exec(); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkSQLAppendWithArgs benchmarks raw INSERT SQL queries with arguments +// using prepared statements. The SQL query looks like the one that is +// generated by upper.io/db. +func BenchmarkSQLPreparedAppendWithArgs(b *testing.B) { + var err error + var sess db.Database + + if sess, err = db.Open(Adapter, settings); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + if _, err = driver.Exec(`DELETE FROM "artist"`); err != nil { + b.Fatal(err) + } + + stmt, err := driver.Prepare(`INSERT INTO "artist" ("name") VALUES(?)`) + + if err != nil { + b.Fatal(err) + } + + args := []interface{}{ + "Hayao Miyazaki", + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err = stmt.Exec(args...); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkSQLAppendWithVariableArgs benchmarks raw INSERT SQL queries with +// arguments using prepared statements. The SQL query looks like the one that +// is generated by upper.io/db. +func BenchmarkSQLPreparedAppendWithVariableArgs(b *testing.B) { + var err error + var sess db.Database + + if sess, err = db.Open(Adapter, settings); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + if _, err = driver.Exec(`DELETE FROM "artist"`); err != nil { + b.Fatal(err) + } + + stmt, err := driver.Prepare(`INSERT INTO "artist" ("name") VALUES(?)`) + + if err != nil { + b.Fatal(err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + args := []interface{}{ + fmt.Sprintf("Hayao Miyazaki %d", rand.Int()), + } + if _, err = stmt.Exec(args...); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkSQLPreparedAppendTransactionWithArgs benchmarks raw INSERT queries +// within a transaction block with arguments and prepared statements. SQL +// queries look like those generated by upper.io/db. +func BenchmarkSQLPreparedAppendTransactionWithArgs(b *testing.B) { + var err error + var sess db.Database + var tx *sqlx.Tx + + if sess, err = db.Open(Adapter, settings); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + if tx, err = driver.Beginx(); err != nil { + b.Fatal(err) + } + + if _, err = tx.Exec(`DELETE FROM "artist"`); err != nil { + b.Fatal(err) + } + + stmt, err := tx.Preparex(`INSERT INTO "artist" ("name") VALUES(?)`) + if err != nil { + b.Fatal(err) + } + + args := []interface{}{ + "Hayao Miyazaki", + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err = stmt.Exec(args...); err != nil { + b.Fatal(err) + } + } + + if err = tx.Commit(); err != nil { + b.Fatal(err) + } +} + +// BenchmarkUpperAppend benchmarks an insertion by upper.io/db. +func BenchmarkUpperAppend(b *testing.B) { + + sess, err := db.Open(Adapter, settings) + if err != nil { + b.Fatal(err) + } + + defer sess.Close() + + artist, err := sess.Collection("artist") + if err != nil { + b.Fatal(err) + } + + artist.Truncate() + + item := struct { + Name string `db:"name"` + }{"Hayao Miyazaki"} + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err = artist.Append(item); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkUpperAppendVariableArgs benchmarks an insertion by upper.io/db +// with variable parameters. +func BenchmarkUpperAppendVariableArgs(b *testing.B) { + + sess, err := db.Open(Adapter, settings) + if err != nil { + b.Fatal(err) + } + + defer sess.Close() + + artist, err := sess.Collection("artist") + if err != nil { + b.Fatal(err) + } + + artist.Truncate() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + item := struct { + Name string `db:"name"` + }{fmt.Sprintf("Hayao Miyazaki %d", rand.Int())} + if _, err = artist.Append(item); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkUpperAppendTransaction benchmarks insertion queries by upper.io/db +// within a transaction operation. +func BenchmarkUpperAppendTransaction(b *testing.B) { + var sess db.Database + var err error + + if sess, err = db.Open(Adapter, settings); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + var tx db.Tx + if tx, err = sess.Transaction(); err != nil { + b.Fatal(err) + } + + var artist db.Collection + if artist, err = tx.Collection("artist"); err != nil { + b.Fatal(err) + } + + if err = artist.Truncate(); err != nil { + b.Fatal(err) + } + + item := struct { + Name string `db:"name"` + }{"Hayao Miyazaki"} + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err = artist.Append(item); err != nil { + b.Fatal(err) + } + } + + if err = tx.Commit(); err != nil { + b.Fatal(err) + } +} + +// BenchmarkUpperAppendTransactionWithMap benchmarks insertion queries by +// upper.io/db within a transaction operation using a map instead of a struct. +func BenchmarkUpperAppendTransactionWithMap(b *testing.B) { + var sess db.Database + var err error + + if sess, err = db.Open(Adapter, settings); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + var tx db.Tx + if tx, err = sess.Transaction(); err != nil { + b.Fatal(err) + } + + var artist db.Collection + if artist, err = tx.Collection("artist"); err != nil { + b.Fatal(err) + } + + if err = artist.Truncate(); err != nil { + b.Fatal(err) + } + + item := map[string]string{ + "name": "Hayao Miyazaki", + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err = artist.Append(item); err != nil { + b.Fatal(err) + } + } + + if err = tx.Commit(); err != nil { + b.Fatal(err) + } +} + +// BenchmarkSQLSelect benchmarks SQL SELECT queries. +func BenchmarkSQLSelect(b *testing.B) { + var err error + var sess db.Database + + if sess, err = connectAndAddFakeRows(); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + var res *sqlx.Rows + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if res, err = driver.Queryx(`SELECT * FROM "artist" WHERE "name" = ?`, artistN(i)); err != nil { + b.Fatal(err) + } + res.Close() + } +} + +// BenchmarkSQLPreparedSelect benchmarks SQL select queries using prepared +// statements. +func BenchmarkSQLPreparedSelect(b *testing.B) { + var err error + var sess db.Database + + if sess, err = connectAndAddFakeRows(); err != nil { + b.Fatal(err) + } + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + stmt, err := driver.Preparex(`SELECT * FROM "artist" WHERE "name" = ?`) + if err != nil { + b.Fatal(err) + } + + var res *sqlx.Rows + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if res, err = stmt.Queryx(artistN(i)); err != nil { + b.Fatal(err) + } + res.Close() + } +} + +// BenchmarkUpperFind benchmarks upper.io/db's One method. +func BenchmarkUpperFind(b *testing.B) { + var err error + var sess db.Database + + if sess, err = connectAndAddFakeRows(); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + artist, err := sess.Collection("artist") + if err != nil { + b.Fatal(err) + } + + type artistType struct { + Name string `db:"name"` + } + + var item artistType + + b.ResetTimer() + for i := 0; i < b.N; i++ { + res := artist.Find(db.Cond{"name": artistN(i)}) + if err = res.One(&item); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkUpperFindAll benchmarks upper.io/db's All method. +func BenchmarkUpperFindAll(b *testing.B) { + var err error + var sess db.Database + + if sess, err = connectAndAddFakeRows(); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + artist, err := sess.Collection("artist") + if err != nil { + b.Fatal(err) + } + + type artistType struct { + Name string `db:"name"` + } + + var items []artistType + + b.ResetTimer() + for i := 0; i < b.N; i++ { + res := artist.Find(db.Or{ + db.Cond{"name": artistN(i)}, + db.Cond{"name": artistN(i + 1)}, + db.Cond{"name": artistN(i + 2)}, + }) + if err = res.All(&items); err != nil { + b.Fatal(err) + } + if len(items) != 3 { + b.Fatal("Expecting 3 results.") + } + } +} + +// BenchmarkSQLUpdate benchmarks SQL UPDATE queries. +func BenchmarkSQLUpdate(b *testing.B) { + var err error + var sess db.Database + + if sess, err = connectAndAddFakeRows(); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err = driver.Exec(`UPDATE "artist" SET "name" = ? WHERE "name" = ?`, updatedArtistN(i), artistN(i)); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkSQLPreparedUpdate benchmarks SQL UPDATE queries. +func BenchmarkSQLPreparedUpdate(b *testing.B) { + var err error + var sess db.Database + + if sess, err = connectAndAddFakeRows(); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + stmt, err := driver.Prepare(`UPDATE "artist" SET "name" = ? WHERE "name" = ?`) + if err != nil { + b.Fatal(err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err = stmt.Exec(updatedArtistN(i), artistN(i)); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkUpperUpdate benchmarks upper.io/db's Update method. +func BenchmarkUpperUpdate(b *testing.B) { + var err error + var sess db.Database + + if sess, err = connectAndAddFakeRows(); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + artist, err := sess.Collection("artist") + if err != nil { + b.Fatal(err) + } + + type artistType struct { + Name string `db:"name"` + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + newValue := artistType{ + Name: updatedArtistN(i), + } + res := artist.Find(db.Cond{"name": artistN(i)}) + if err = res.Update(newValue); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkSQLDelete benchmarks SQL DELETE queries. +func BenchmarkSQLDelete(b *testing.B) { + var err error + var sess db.Database + + if sess, err = connectAndAddFakeRows(); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err = driver.Exec(`DELETE FROM "artist" WHERE "name" = ?`, artistN(i)); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkSQLPreparedDelete benchmarks SQL DELETE queries. +func BenchmarkSQLPreparedDelete(b *testing.B) { + var err error + var sess db.Database + + if sess, err = connectAndAddFakeRows(); err != nil { + b.Fatal(err) + } + defer sess.Close() + + driver := sess.Driver().(*sqlx.DB) + + stmt, err := driver.Prepare(`DELETE FROM "artist" WHERE "name" = ?`) + if err != nil { + b.Fatal(err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err = stmt.Exec(artistN(i)); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkUpperRemove benchmarks +func BenchmarkUpperRemove(b *testing.B) { + var err error + var sess db.Database + + if sess, err = connectAndAddFakeRows(); err != nil { + b.Fatal(err) + } + + defer sess.Close() + + artist, err := sess.Collection("artist") + if err != nil { + b.Fatal(err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + res := artist.Find(db.Cond{"name": artistN(i)}) + if err = res.Remove(); err != nil { + b.Fatal(err) + } + } +} diff --git a/sqlite/collection.go b/sqlite/collection.go index d7fe183a910cfd7d88491e5a3a5b2673eb7f5635..59442646ce2584454989604263a1540a8656f01f 100644 --- a/sqlite/collection.go +++ b/sqlite/collection.go @@ -47,7 +47,7 @@ func (t *table) Find(terms ...interface{}) db.Result { // Truncate deletes all rows from the table. func (t *table) Truncate() error { - _, err := t.database.Exec(sqlgen.Statement{ + _, err := t.database.Exec(&sqlgen.Statement{ Type: sqlgen.Truncate, Table: sqlgen.TableWithName(t.MainTableName()), }) @@ -81,7 +81,7 @@ func (t *table) Append(item interface{}) (interface{}, error) { } } - stmt := sqlgen.Statement{ + stmt := &sqlgen.Statement{ Type: sqlgen.Insert, Table: sqlgen.TableWithName(t.MainTableName()), Columns: sqlgenCols, diff --git a/sqlite/database.go b/sqlite/database.go index f449b976081f6fc5718e00a872804382f5a3e345..c75247d364220f541c16673e62f4ec2306e181e6 100644 --- a/sqlite/database.go +++ b/sqlite/database.go @@ -29,6 +29,7 @@ import ( "github.com/jmoiron/sqlx" _ "github.com/mattn/go-sqlite3" // SQLite3 driver. + "upper.io/cache" "upper.io/db" "upper.io/db/util/schema" "upper.io/db/util/sqlgen" @@ -48,7 +49,8 @@ type database struct { // columns property was introduced so we could query PRAGMA data only once // and retrieve all the column information we'd need, such as name and if it // is a primary key. - columns map[string][]columnSchemaT + columns map[string][]columnSchemaT + cachedStatements *cache.Cache } type tx struct { @@ -56,6 +58,11 @@ type tx struct { *database } +type cachedStatement struct { + *sqlx.Stmt + query string +} + var ( _ = db.Database(&database{}) _ = db.Tx(&tx{}) @@ -66,6 +73,40 @@ type columnSchemaT struct { PK int `db:"pk"` } +func (d *database) prepareStatement(stmt *sqlgen.Statement) (p *sqlx.Stmt, query string, err error) { + if d.session == nil { + return nil, "", db.ErrNotConnected + } + + pc, ok := d.cachedStatements.ReadRaw(stmt) + + if ok { + ps := pc.(*cachedStatement) + p = ps.Stmt + query = ps.query + } else { + query = compileAndReplacePlaceholders(stmt) + + if d.tx != nil { + p, err = d.tx.Preparex(query) + } else { + p, err = d.session.Preparex(query) + } + + if err != nil { + return nil, "", err + } + + d.cachedStatements.Write(stmt, &cachedStatement{p, query}) + } + + return p, query, nil +} + +func compileAndReplacePlaceholders(stmt *sqlgen.Statement) string { + return stmt.Compile(template.Template) +} + // Driver returns the underlying *sqlx.DB instance. func (d *database) Driver() interface{} { return d.session @@ -96,8 +137,12 @@ func (d *database) Open() error { d.session.Mapper = sqlutil.NewMapper() - if err = d.populateSchema(); err != nil { - return err + d.cachedStatements = cache.NewCache() + + if d.schema == nil { + if err = d.populateSchema(); err != nil { + return err + } } return nil @@ -110,14 +155,13 @@ func (d *database) Clone() (db.Database, error) { } func (d *database) clone() (*database, error) { - src := &database{} - src.Setup(d.connURL) - - if err := src.Open(); err != nil { + clone := &database{ + schema: d.schema, + } + if err := clone.Setup(d.connURL); err != nil { return nil, err } - - return src, nil + return clone, nil } // Ping checks whether a connection to the database is still alive by pinging @@ -187,7 +231,7 @@ func (d *database) Collections() (collections []string, err error) { // Schema is empty. // Querying table names. - stmt := sqlgen.Statement{ + stmt := &sqlgen.Statement{ Type: sqlgen.Select, Columns: sqlgen.JoinColumns( sqlgen.ColumnWithName(`tbl_name`), @@ -231,24 +275,26 @@ func (d *database) Collections() (collections []string, err error) { } // Use changes the active database. -func (d *database) Use(database string) (err error) { +func (d *database) Use(name string) (err error) { var conn ConnectionURL if conn, err = ParseURL(d.connURL.String()); err != nil { return err } - conn.Database = database + conn.Database = name d.connURL = conn + d.schema = nil + return d.Open() } // Drop removes all tables from the current database. func (d *database) Drop() error { - _, err := d.Query(sqlgen.Statement{ + _, err := d.Query(&sqlgen.Statement{ Type: sqlgen.DropDatabase, Database: sqlgen.DatabaseWithName(d.schema.Name), }) @@ -288,90 +334,72 @@ func (d *database) Transaction() (db.Tx, error) { } // Exec compiles and executes a statement that does not return any rows. -func (d *database) Exec(stmt sqlgen.Statement, args ...interface{}) (sql.Result, error) { +func (d *database) Exec(stmt *sqlgen.Statement, args ...interface{}) (sql.Result, error) { var query string - var res sql.Result + var p *sqlx.Stmt var err error - var start, end int64 - start = time.Now().UnixNano() - - defer func() { - end = time.Now().UnixNano() - sqlutil.Log(query, args, err, start, end) - }() + if db.Debug { + var start, end int64 + start = time.Now().UnixNano() - if d.session == nil { - return nil, db.ErrNotConnected + defer func() { + end = time.Now().UnixNano() + sqlutil.Log(query, args, err, start, end) + }() } - query = stmt.Compile(template.Template) - - if d.tx != nil { - res, err = d.tx.Exec(query, args...) - } else { - res, err = d.session.Exec(query, args...) + if p, query, err = d.prepareStatement(stmt); err != nil { + return nil, err } - return res, err + return p.Exec(args...) } // Query compiles and executes a statement that returns rows. -func (d *database) Query(stmt sqlgen.Statement, args ...interface{}) (*sqlx.Rows, error) { - var rows *sqlx.Rows +func (d *database) Query(stmt *sqlgen.Statement, args ...interface{}) (*sqlx.Rows, error) { var query string + var p *sqlx.Stmt var err error - var start, end int64 - start = time.Now().UnixNano() + if db.Debug { + var start, end int64 + start = time.Now().UnixNano() - defer func() { - end = time.Now().UnixNano() - sqlutil.Log(query, args, err, start, end) - }() - - if d.session == nil { - return nil, db.ErrNotConnected + defer func() { + end = time.Now().UnixNano() + sqlutil.Log(query, args, err, start, end) + }() } - query = stmt.Compile(template.Template) - - if d.tx != nil { - rows, err = d.tx.Queryx(query, args...) - } else { - rows, err = d.session.Queryx(query, args...) + if p, query, err = d.prepareStatement(stmt); err != nil { + return nil, err } - return rows, err + return p.Queryx(args...) } // QueryRow compiles and executes a statement that returns at most one row. -func (d *database) QueryRow(stmt sqlgen.Statement, args ...interface{}) (*sqlx.Row, error) { +func (d *database) QueryRow(stmt *sqlgen.Statement, args ...interface{}) (*sqlx.Row, error) { var query string - var row *sqlx.Row + var p *sqlx.Stmt var err error - var start, end int64 - start = time.Now().UnixNano() + if db.Debug { + var start, end int64 + start = time.Now().UnixNano() - defer func() { - end = time.Now().UnixNano() - sqlutil.Log(query, args, err, start, end) - }() - - if d.session == nil { - return nil, db.ErrNotConnected + defer func() { + end = time.Now().UnixNano() + sqlutil.Log(query, args, err, start, end) + }() } - query = stmt.Compile(template.Template) - - if d.tx != nil { - row = d.tx.QueryRowx(query, args...) - } else { - row = d.session.QueryRowx(query, args...) + if p, query, err = d.prepareStatement(stmt); err != nil { + return nil, err } - return row, err + return p.QueryRowx(args...), nil } // populateSchema looks up for the table info in the database and populates its @@ -405,7 +433,7 @@ func (d *database) populateSchema() (err error) { } func (d *database) tableExists(names ...string) error { - var stmt sqlgen.Statement + var stmt *sqlgen.Statement var err error var rows *sqlx.Rows @@ -416,7 +444,7 @@ func (d *database) tableExists(names ...string) error { continue } - stmt = sqlgen.Statement{ + stmt = &sqlgen.Statement{ Type: sqlgen.Select, Table: sqlgen.TableWithName(`sqlite_master`), Columns: sqlgen.JoinColumns( diff --git a/sqlite/database_test.go b/sqlite/database_test.go index 0befae10c9d875deb08f52f193021a20391ac6f0..6d616aef1799e00a24d224294df3e55ea5913f7d 100644 --- a/sqlite/database_test.go +++ b/sqlite/database_test.go @@ -1361,180 +1361,3 @@ func TestDataTypes(t *testing.T) { t.Fatalf("Struct is different.") } } - -// Benchmarking raw database/sql. -func BenchmarkAppendRawSQL(b *testing.B) { - var err error - var sess db.Database - - if sess, err = db.Open(Adapter, settings); err != nil { - b.Fatal(err) - } - - defer sess.Close() - - driver := sess.Driver().(*sqlx.DB) - - if _, err = driver.Exec(`DELETE FROM "artist"`); err != nil { - b.Fatal(err) - } - - b.ResetTimer() - stmt, err := driver.Prepare( - `INSERT INTO "artist" ("name") VALUES('Hayao Miyazaki')`) - if err != nil { - b.Fatal(err) - } - for i := 0; i < b.N; i++ { - if _, err = stmt.Exec(); err != nil { - b.Fatal(err) - } - } -} - -// Benchmarking Append(). -// -// Contributed by wei2912 -// See: https://github.com/gosexy/db/issues/20#issuecomment-20097801 -func BenchmarkAppendUpper(b *testing.B) { - sess, err := db.Open(Adapter, settings) - - if err != nil { - b.Fatal(err) - } - - defer sess.Close() - - artist, err := sess.Collection("artist") - artist.Truncate() - - item := struct { - Name string `db:"name"` - }{"Hayao Miyazaki"} - - b.ResetTimer() - for i := 0; i < b.N; i++ { - if _, err = artist.Append(item); err != nil { - b.Fatal(err) - } - } -} - -// Benchmarking raw database/sql. -func BenchmarkAppendTxRawSQL(b *testing.B) { - var err error - var sess db.Database - var tx *sql.Tx - - if sess, err = db.Open(Adapter, settings); err != nil { - b.Fatal(err) - } - - defer sess.Close() - - driver := sess.Driver().(*sqlx.DB) - - if tx, err = driver.Begin(); err != nil { - b.Fatal(err) - } - - if _, err = tx.Exec(`DELETE FROM "artist"`); err != nil { - b.Fatal(err) - } - - b.ResetTimer() - stmt, err := tx.Prepare( - `INSERT INTO "artist" ("name") VALUES('Hayao Miyazaki')`) - if err != nil { - b.Fatal(err) - } - for i := 0; i < b.N; i++ { - if _, err = stmt.Exec(); err != nil { - b.Fatal(err) - } - } - - if err = tx.Commit(); err != nil { - b.Fatal(err) - } -} - -// Benchmarking Append() with transactions. -func BenchmarkAppendTxUpper(b *testing.B) { - var sess db.Database - var err error - - if sess, err = db.Open(Adapter, settings); err != nil { - b.Fatal(err) - } - - defer sess.Close() - - var tx db.Tx - if tx, err = sess.Transaction(); err != nil { - b.Fatal(err) - } - - var artist db.Collection - if artist, err = tx.Collection("artist"); err != nil { - b.Fatal(err) - } - - if err = artist.Truncate(); err != nil { - b.Fatal(err) - } - - item := struct { - Name string `db:"name"` - }{"Hayao Miyazaki"} - - b.ResetTimer() - for i := 0; i < b.N; i++ { - if _, err = artist.Append(item); err != nil { - b.Fatal(err) - } - } - - if err = tx.Commit(); err != nil { - b.Fatal(err) - } -} - -// Benchmarking Append() with map. -func BenchmarkAppendTxUpperMap(b *testing.B) { - var sess db.Database - var err error - - if sess, err = db.Open(Adapter, settings); err != nil { - b.Fatal(err) - } - - defer sess.Close() - - var tx db.Tx - if tx, err = sess.Transaction(); err != nil { - b.Fatal(err) - } - - var artist db.Collection - if artist, err = tx.Collection("artist"); err != nil { - b.Fatal(err) - } - - if err = artist.Truncate(); err != nil { - b.Fatal(err) - } - - item := map[string]string{"name": "Hayao Miyazaki"} - - b.ResetTimer() - for i := 0; i < b.N; i++ { - if _, err = artist.Append(item); err != nil { - b.Fatal(err) - } - } - - if err = tx.Commit(); err != nil { - b.Fatal(err) - } -} diff --git a/util/sqlutil/debug.go b/util/sqlutil/debug.go index ee9ec9759fd302c225badc703ba346093797f2f9..3c997891db4a50bf7d59572c60c2bbbffff31615 100644 --- a/util/sqlutil/debug.go +++ b/util/sqlutil/debug.go @@ -30,6 +30,12 @@ import ( "upper.io/db" ) +func init() { + if os.Getenv(db.EnvEnableDebug) != "" { + db.Debug = true + } +} + // Debug is used for printing SQL queries and arguments. type Debug struct { SQL string @@ -63,15 +69,8 @@ func (d *Debug) Print() { log.Printf("\n\t%s\n\n", strings.Join(s, "\n\t")) } -func IsDebugEnabled() bool { - if os.Getenv(db.EnvEnableDebug) != "" { - return true - } - return false -} - func Log(query string, args []interface{}, err error, start int64, end int64) { - if IsDebugEnabled() { + if db.Debug { d := Debug{query, args, err, start, end} d.Print() } diff --git a/util/sqlutil/result/result.go b/util/sqlutil/result/result.go index f570a01bb47f0af734419d44eeda2f44b5c2d0a7..64c208a5252cbcabc2fed8b4347cd46e4e5c68f9 100644 --- a/util/sqlutil/result/result.go +++ b/util/sqlutil/result/result.go @@ -68,7 +68,7 @@ func (r *Result) setCursor() error { var err error // We need a cursor, if the cursor does not exists yet then we create one. if r.cursor == nil { - r.cursor, err = r.table.Query(sqlgen.Statement{ + r.cursor, err = r.table.Query(&sqlgen.Statement{ Type: sqlgen.Select, Table: sqlgen.TableWithName(r.table.Name()), Columns: &r.columns, @@ -249,7 +249,7 @@ func (r *Result) Next(dst interface{}) (err error) { func (r *Result) Remove() error { var err error - _, err = r.table.Exec(sqlgen.Statement{ + _, err = r.table.Exec(&sqlgen.Statement{ Type: sqlgen.Delete, Table: sqlgen.TableWithName(r.table.Name()), Where: &r.where, @@ -276,7 +276,7 @@ func (r *Result) Update(values interface{}) error { vv = append(vv, r.arguments...) - _, err = r.table.Exec(sqlgen.Statement{ + _, err = r.table.Exec(&sqlgen.Statement{ Type: sqlgen.Update, Table: sqlgen.TableWithName(r.table.Name()), ColumnValues: cvs, @@ -299,7 +299,7 @@ func (r *Result) Close() (err error) { func (r *Result) Count() (uint64, error) { var count counter - row, err := r.table.QueryRow(sqlgen.Statement{ + row, err := r.table.QueryRow(&sqlgen.Statement{ Type: sqlgen.Count, Table: sqlgen.TableWithName(r.table.Name()), Where: &r.where, diff --git a/util/sqlutil/result/table.go b/util/sqlutil/result/table.go index 57ec42ac1bdf8abe90ea0667e7b8177089820c8d..12076d80629c02daec0635f3e29e40a86cd023de 100644 --- a/util/sqlutil/result/table.go +++ b/util/sqlutil/result/table.go @@ -8,8 +8,8 @@ import ( type DataProvider interface { Name() string - Query(sqlgen.Statement, ...interface{}) (*sqlx.Rows, error) - QueryRow(sqlgen.Statement, ...interface{}) (*sqlx.Row, error) - Exec(sqlgen.Statement, ...interface{}) (sql.Result, error) + Query(*sqlgen.Statement, ...interface{}) (*sqlx.Rows, error) + QueryRow(*sqlgen.Statement, ...interface{}) (*sqlx.Row, error) + Exec(*sqlgen.Statement, ...interface{}) (sql.Result, error) FieldValues(interface{}) ([]string, []interface{}, error) }