diff --git a/.travis.yml b/.travis.yml index 512b33a3..6e027af6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,8 @@ sudo: required go: - 1.4 - 1.5 + - 1.6 + - 1.7 services: - docker diff --git a/Makefile b/Makefile index 9cee255b..eaf7e122 100644 --- a/Makefile +++ b/Makefile @@ -1,15 +1,23 @@ +TESTFLAGS?= IMAGE=mattes/migrate DCR=docker-compose run --rm -.PHONY: clean test build release docker-build docker-push run +GOTEST=go test $(TESTFLAGS) `go list ./... | grep -v "/vendor/"` +.PHONY: clean test build release docker-build docker-push run all: release clean: rm -f migrate -test: +fmt: + @gofmt -s -w `go list -f {{.Dir}} ./... | grep -v "/vendor/"` + +test: fmt $(DCR) go-test +go-test: fmt + @$(GOTEST) + build: $(DCR) go-build diff --git a/README.md b/README.md index 415a3aff..bb4dc738 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ __Features__ * [Cassandra](https://github.com/mattes/migrate/tree/master/driver/cassandra) * [SQLite](https://github.com/mattes/migrate/tree/master/driver/sqlite3) * [MySQL](https://github.com/mattes/migrate/tree/master/driver/mysql) ([experimental](https://github.com/mattes/migrate/issues/1#issuecomment-58728186)) + * [Ql](https://github.com/mattes/migrate/tree/master/driver/ql) * Bash (planned) Need another driver? Just implement the [Driver interface](http://godoc.org/github.com/mattes/migrate/driver#Driver) and open a PR. @@ -100,15 +101,15 @@ go migrate.Up(pipe, "driver://url", "./path") The format of migration files looks like this: ``` -001_initial_plan_to_do_sth.up.sql # up migration instructions -001_initial_plan_to_do_sth.down.sql # down migration instructions -002_xxx.up.sql -002_xxx.down.sql +1481574547_initial_plan_to_do_sth.up.sql # up migration instructions +1481574547_initial_plan_to_do_sth.down.sql # down migration instructions +1482438365_xxx.up.sql +1482438365_xxx.down.sql ... ``` Why two files? This way you could still do sth like -``psql -f ./db/migrations/001_initial_plan_to_do_sth.up.sql`` and there is no +``psql -f ./db/migrations/1481574547_initial_plan_to_do_sth.up.sql`` and there is no need for any custom markup language to divide up and down migrations. Please note that the filename extension depends on the driver. diff --git a/docker-compose.yml b/docker-compose.yml index cac3abcf..8dcd17c5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,6 +10,8 @@ go-test: - postgres - mysql - cassandra + - crate + - mongo go-build: <<: *go command: sh -c 'go get -v && go build -ldflags ''-s'' -o migrater' @@ -21,6 +23,12 @@ mysql: image: mysql environment: MYSQL_DATABASE: migratetest - MYSQL_ALLOW_EMPTY_PASSWORD: yes + MYSQL_ALLOW_EMPTY_PASSWORD: "yes" cassandra: image: cassandra:2.2 +crate: + image: crate +mongo: + image: mongo:3.2.6 + + diff --git a/driver/cassandra/README.md b/driver/cassandra/README.md index 69596069..75315bfa 100644 --- a/driver/cassandra/README.md +++ b/driver/cassandra/README.md @@ -8,8 +8,13 @@ migrate -url cassandra://host:port/keyspace -path ./db/migrations up migrate help # for more info ``` +Url format +- Authentication: `cassandra://username:password@host:port/keyspace` +- Cassandra v3.x: `cassandra://host:port/keyspace?protocol=4` + + ## Authors * Paul Bergeron, https://github.com/dinedal * Johnny Bergström, https://github.com/balboah -* pateld982, http://github.com/pateld982 \ No newline at end of file +* pateld982, http://github.com/pateld982 diff --git a/driver/cassandra/cassandra.go b/driver/cassandra/cassandra.go index e242de2d..eb66174e 100644 --- a/driver/cassandra/cassandra.go +++ b/driver/cassandra/cassandra.go @@ -41,18 +41,36 @@ const ( ) // Cassandra Driver URL format: -// cassandra://host:port/keyspace?protocol=version +// cassandra://host:port/keyspace?protocol=version&consistency=level // -// Example: +// Examples: // cassandra://localhost/SpaceOfKeys?protocol=4 +// cassandra://localhost/SpaceOfKeys?protocol=4&consistency=all +// cassandra://localhost/SpaceOfKeys?consistency=quorum func (driver *Driver) Initialize(rawurl string) error { u, err := url.Parse(rawurl) + if err != nil { + return fmt.Errorf("failed to parse connectil url: %v", err) + } + + if u.Path == "" { + return fmt.Errorf("no keyspace provided in connection url") + } cluster := gocql.NewCluster(u.Host) cluster.Keyspace = u.Path[1:len(u.Path)] cluster.Consistency = gocql.All cluster.Timeout = 1 * time.Minute + if len(u.Query().Get("consistency")) > 0 { + consistency, err := parseConsistency(u.Query().Get("consistency")) + if err != nil { + return err + } + + cluster.Consistency = consistency + } + if len(u.Query().Get("protocol")) > 0 { protoversion, err := strconv.Atoi(u.Query().Get("protocol")) if err != nil { @@ -169,3 +187,20 @@ func (driver *Driver) Version() (uint64, error) { func init() { driver.RegisterDriver("cassandra", &Driver{}) } + +// ParseConsistency wraps gocql.ParseConsistency to return an error +// instead of a panicing. +func parseConsistency(consistencyStr string) (consistency gocql.Consistency, err error) { + defer func() { + if r := recover(); r != nil { + var ok bool + err, ok = r.(error) + if !ok { + err = fmt.Errorf("Failed to parse consistency \"%s\": %v", consistencyStr, r) + } + } + }() + consistency = gocql.ParseConsistency(consistencyStr) + + return consistency, nil +} diff --git a/driver/cassandra/cassandra_test.go b/driver/cassandra/cassandra_test.go index ea737945..202d5e74 100644 --- a/driver/cassandra/cassandra_test.go +++ b/driver/cassandra/cassandra_test.go @@ -13,6 +13,10 @@ import ( ) func TestMigrate(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } + var session *gocql.Session host := os.Getenv("CASSANDRA_PORT_9042_TCP_ADDR") @@ -112,5 +116,35 @@ func TestMigrate(t *testing.T) { if err := d.Close(); err != nil { t.Fatal(err) } +} + +func TestInitializeReturnsErrorsForBadUrls(t *testing.T) { + var session *gocql.Session + + host := os.Getenv("CASSANDRA_PORT_9042_TCP_ADDR") + port := os.Getenv("CASSANDRA_PORT_9042_TCP_PORT") + + cluster := gocql.NewCluster(host) + cluster.Consistency = gocql.All + cluster.Timeout = 1 * time.Minute + + session, err := cluster.CreateSession() + if err != nil { + t.Fatal(err) + } + defer session.Close() + if err := session.Query(`CREATE KEYSPACE IF NOT EXISTS migrate WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor': 1};`).Exec(); err != nil { + t.Fatal(err) + } + d := &Driver{} + invalidURL := "sdf://asdf://as?df?a" + if err := d.Initialize(invalidURL); err == nil { + t.Errorf("expected an error to be returned if url could not be parsed") + } + + noKeyspace := "cassandra://" + host + ":" + port + if err := d.Initialize(noKeyspace); err == nil { + t.Errorf("expected an error to be returned if no keyspace provided") + } } diff --git a/driver/crate/README.md b/driver/crate/README.md new file mode 100644 index 00000000..a43d0133 --- /dev/null +++ b/driver/crate/README.md @@ -0,0 +1,15 @@ +# Crate driver + +This is a driver for the [Crate](https://crate.io) database. It is based on the Crate +sql driver by [herenow](https://github.com/herenow/go-crate). + +This driver does not use transactions! This is not a limitation of the driver, but a +limitation of Crate. So handle situations with failed migrations with care! + +## Usage + +```bash +migrate -url http://host:port -path ./db/migrations create add_field_to_table +migrate -url http://host:port -path ./db/migrations up +migrate help # for more info +``` \ No newline at end of file diff --git a/driver/crate/crate.go b/driver/crate/crate.go new file mode 100644 index 00000000..eb30853a --- /dev/null +++ b/driver/crate/crate.go @@ -0,0 +1,115 @@ +// Package crate implements a driver for the Crate.io database +package crate + +import ( + "database/sql" + "fmt" + "strings" + + _ "github.com/herenow/go-crate" + "github.com/mattes/migrate/driver" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" +) + +func init() { + driver.RegisterDriver("crate", &Driver{}) +} + +type Driver struct { + db *sql.DB +} + +const tableName = "schema_migrations" + +func (driver *Driver) Initialize(url string) error { + url = strings.Replace(url, "crate", "http", 1) + db, err := sql.Open("crate", url) + if err != nil { + return err + } + if err := db.Ping(); err != nil { + return err + } + driver.db = db + + if err := driver.ensureVersionTableExists(); err != nil { + return err + } + return nil +} + +func (driver *Driver) Close() error { + if err := driver.db.Close(); err != nil { + return err + } + return nil +} + +func (driver *Driver) FilenameExtension() string { + return "sql" +} + +func (driver *Driver) Version() (uint64, error) { + var version uint64 + err := driver.db.QueryRow("SELECT version FROM " + tableName + " ORDER BY version DESC LIMIT 1").Scan(&version) + switch { + case err == sql.ErrNoRows: + return 0, nil + case err != nil: + return 0, err + default: + return version, nil + } +} + +func (driver *Driver) Migrate(f file.File, pipe chan interface{}) { + defer close(pipe) + pipe <- f + + if err := f.ReadContent(); err != nil { + pipe <- err + return + } + + lines := splitContent(string(f.Content)) + for _, line := range lines { + _, err := driver.db.Exec(line) + if err != nil { + pipe <- err + return + } + } + + if f.Direction == direction.Up { + if _, err := driver.db.Exec("INSERT INTO "+tableName+" (version) VALUES (?)", f.Version); err != nil { + pipe <- err + return + } + } else if f.Direction == direction.Down { + if _, err := driver.db.Exec("DELETE FROM "+tableName+" WHERE version=?", f.Version); err != nil { + pipe <- err + return + } + } +} + +func splitContent(content string) []string { + lines := strings.Split(content, ";") + resultLines := make([]string, 0, len(lines)) + for i, line := range lines { + line = strings.Replace(lines[i], ";", "", -1) + line = strings.TrimSpace(line) + if line != "" { + resultLines = append(resultLines, line) + } + } + return resultLines +} + +func (driver *Driver) ensureVersionTableExists() error { + if _, err := driver.db.Exec(fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (version INTEGER PRIMARY KEY)", tableName)); err != nil { + return err + } + return nil +} diff --git a/driver/crate/crate_test.go b/driver/crate/crate_test.go new file mode 100644 index 00000000..164cd57a --- /dev/null +++ b/driver/crate/crate_test.go @@ -0,0 +1,111 @@ +package crate + +import ( + "fmt" + "os" + "testing" + + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" + pipep "github.com/mattes/migrate/pipe" +) + +func TestContentSplit(t *testing.T) { + content := `CREATE TABLE users (user_id STRING primary key, first_name STRING, last_name STRING, email STRING, password_hash STRING) CLUSTERED INTO 3 shards WITH (number_of_replicas = 0); +CREATE TABLE units (unit_id STRING primary key, name STRING, members array(string)) CLUSTERED INTO 3 shards WITH (number_of_replicas = 0); +CREATE TABLE available_connectors (technology_id STRING primary key, description STRING, icon STRING, link STRING, configuration_parameters array(object as (name STRING, type STRING))) CLUSTERED INTO 3 shards WITH (number_of_replicas = 0); + ` + + lines := splitContent(content) + if len(lines) != 3 { + t.Errorf("Expected 3 lines, but got %d", len(lines)) + } + + if lines[0] != "CREATE TABLE users (user_id STRING primary key, first_name STRING, last_name STRING, email STRING, password_hash STRING) CLUSTERED INTO 3 shards WITH (number_of_replicas = 0)" { + t.Error("Line does not match expected output") + } + + if lines[1] != "CREATE TABLE units (unit_id STRING primary key, name STRING, members array(string)) CLUSTERED INTO 3 shards WITH (number_of_replicas = 0)" { + t.Error("Line does not match expected output") + } + + if lines[2] != "CREATE TABLE available_connectors (technology_id STRING primary key, description STRING, icon STRING, link STRING, configuration_parameters array(object as (name STRING, type STRING))) CLUSTERED INTO 3 shards WITH (number_of_replicas = 0)" { + t.Error("Line does not match expected output") + } +} + +func TestMigrate(t *testing.T) { + host := os.Getenv("CRATE_PORT_4200_TCP_ADDR") + port := os.Getenv("CRATE_PORT_4200_TCP_PORT") + + url := fmt.Sprintf("crate://%s:%s", host, port) + + driver := &Driver{} + + if err := driver.Initialize(url); err != nil { + t.Fatal(err) + } + + successFiles := []file.File{ + { + Path: "/foobar", + FileName: "001_foobar.up.sql", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + CREATE TABLE yolo ( + id integer primary key, + msg string + ); + `), + }, + { + Path: "/foobar", + FileName: "002_foobar.down.sql", + Version: 1, + Name: "foobar", + Direction: direction.Down, + Content: []byte(` + DROP TABLE yolo; + `), + }, + } + + failFiles := []file.File{ + { + Path: "/foobar", + FileName: "002_foobar.up.sql", + Version: 2, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + CREATE TABLE error ( + id THIS WILL CAUSE AN ERROR + ) + `), + }, + } + + for _, file := range successFiles { + pipe := pipep.New() + go driver.Migrate(file, pipe) + errs := pipep.ReadErrors(pipe) + if len(errs) > 0 { + t.Fatal(errs) + } + } + + for _, file := range failFiles { + pipe := pipep.New() + go driver.Migrate(file, pipe) + errs := pipep.ReadErrors(pipe) + if len(errs) == 0 { + t.Fatal("Migration should have failed but succeeded") + } + } + + if err := driver.Close(); err != nil { + t.Fatal(err) + } +} diff --git a/driver/mongodb/README.md b/driver/mongodb/README.md new file mode 100644 index 00000000..d7d8351d --- /dev/null +++ b/driver/mongodb/README.md @@ -0,0 +1,104 @@ +# MongoDB Driver + +* Runs pre-registered Golang methods that receive a single `*mgo.Session` parameter and return `error` on failure. +* Stores migration version details in collection ``db_migrations``. + This collection will be auto-generated. +* Migrations do not run in transactions, there are no built-in transactions in MongoDB. + That means that if a migration fails, it will not be rolled back. +* There is no out-of-the-box support for command-line interface via terminal. + +## Usage in Go + +```go +import "github.com/mattes/migrate/migrate" + +// Import your migration methods package so that they are registered and available for the MongoDB driver. +// There is no need to import the MongoDB driver explicitly, as it should already be imported by your migration methods package. +import _ "my_mongo_db_migrator" + +// use synchronous versions of migration functions ... +allErrors, ok := migrate.UpSync("mongodb://host:port", "./path") +if !ok { + fmt.Println("Oh no ...") + // do sth with allErrors slice +} + +// use the asynchronous version of migration functions ... +pipe := migrate.NewPipe() +go migrate.Up(pipe, "mongodb://host:port", "./path") +// pipe is basically just a channel +// write your own channel listener. see writePipe() in main.go as an example. +``` + +## Migration files format + +The migration files should have an ".mgo" extension and contain a list of registered methods names. + +Migration methods should satisfy the following: +* They should be exported (their name should start with a capital letter) +* Their type should be `func (*mgo.Session) error` + +Recommended (but not required) naming conventions for migration methods: +* Prefix with V : for example V001 for version 1. +* Suffix with "_up" or "_down" for up and down migrations correspondingly. + +001_first_release.up.mgo +``` +V001_some_migration_operation_up +V001_some_other_operation_up +... +``` + +001_first_release.down.mgo +``` +V001_some_other_operation_down +V001_some_migration_operation_down +... +``` + + +## Methods registration + +For a detailed example see: [sample_mongodb_migrator.go](https://github.com/mattes/migrate/blob/master/driver/mongodb/example/sample_mongdb_migrator.go) + +```go +package my_mongo_db_migrator + +import ( + "github.com/mattes/migrate/driver/mongodb" + "github.com/mattes/migrate/driver/mongodb/gomethods" + "gopkg.in/mgo.v2" +) + +// common boilerplate +type MyMongoDbMigrator struct { +} + +func (r *MyMongoDbMigrator) DbName() string { + return "" +} + +var _ mongodb.MethodsReceiver = (*MyMongoDbMigrator)(nil) + +func init() { + gomethods.RegisterMethodsReceiverForDriver("mongodb", &MyMongoDbMigrator{}) +} + + +// Here goes the application-specific migration logic +func (r *MyMongoDbMigrator) V001_some_migration_operation_up(session *mgo.Session) error { + // do something + return nil +} + +func (r *MyMongoDbMigrator) V001_some_migration_operation_down(session *mgo.Session) error { + // revert some_migration_operation_up from above + return nil +} + +``` + +## Authors + +* Demitry Gershovich, https://github.com/dimag-jfrog + diff --git a/driver/mongodb/example/mongodb_test.go b/driver/mongodb/example/mongodb_test.go new file mode 100644 index 00000000..86c37c00 --- /dev/null +++ b/driver/mongodb/example/mongodb_test.go @@ -0,0 +1,310 @@ +package example + +import ( + "testing" + + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" + + "github.com/mattes/migrate/driver" + "github.com/mattes/migrate/driver/mongodb" + "github.com/mattes/migrate/driver/mongodb/gomethods" + pipep "github.com/mattes/migrate/pipe" + "os" + "reflect" + "time" +) + +type ExpectedMigrationResult struct { + Organizations []Organization + Organizations_v2 []Organization_v2 + Users []User + Errors []error +} + +func RunMigrationAndAssertResult( + t *testing.T, + title string, + d *mongodb.Driver, + file file.File, + expected *ExpectedMigrationResult) { + + actualOrganizations := []Organization{} + actualOrganizations_v2 := []Organization_v2{} + actualUsers := []User{} + var err error + var pipe chan interface{} + var errs []error + + pipe = pipep.New() + go d.Migrate(file, pipe) + errs = pipep.ReadErrors(pipe) + + session := d.Session + if len(expected.Organizations) > 0 { + err = session.DB(DB_NAME).C(ORGANIZATIONS_C).Find(nil).All(&actualOrganizations) + } else { + err = session.DB(DB_NAME).C(ORGANIZATIONS_C).Find(nil).All(&actualOrganizations_v2) + } + if err != nil { + t.Fatal("Failed to query Organizations collection") + } + + err = session.DB(DB_NAME).C(USERS_C).Find(nil).All(&actualUsers) + if err != nil { + t.Fatal("Failed to query Users collection") + } + + if !reflect.DeepEqual(expected.Errors, errs) { + t.Fatalf("Migration '%s': FAILED\nexpected errors %v\nbut got %v", title, expected.Errors, errs) + } + + if !reflect.DeepEqual(expected.Organizations, actualOrganizations) { + t.Fatalf("Migration '%s': FAILED\nexpected organizations %v\nbut got %v", title, expected.Organizations, actualOrganizations) + } + + if !reflect.DeepEqual(expected.Organizations_v2, actualOrganizations_v2) { + t.Fatalf("Migration '%s': FAILED\nexpected organizations v2 %v\nbut got %v", title, expected.Organizations_v2, actualOrganizations_v2) + } + + if !reflect.DeepEqual(expected.Users, actualUsers) { + t.Fatalf("Migration '%s': FAILED\nexpected users %v\nbut got %v", title, expected.Users, actualUsers) + + } + // t.Logf("Migration '%s': PASSED", title) +} + +func TestMigrate(t *testing.T) { + defer func() { + if r := recover(); r != nil { + t.Fatalf("Test failed on panic: %v", r) + } + }() + + host := os.Getenv("MONGO_PORT_27017_TCP_ADDR") + port := os.Getenv("MONGO_PORT_27017_TCP_PORT") + + driverUrl := "mongodb://" + host + ":" + port + + d0 := driver.GetDriver("mongodb") + d, ok := d0.(*mongodb.Driver) + if !ok { + t.Fatal("MongoDbGoMethodsDriver has not registered") + } + + if err := d.Initialize(driverUrl); err != nil { + t.Fatal(err) + } + + // Reset DB + d.Session.DB(DB_NAME).C(mongodb.MIGRATE_C).DropCollection() + d.Session.DB(DB_NAME).C(ORGANIZATIONS_C).DropCollection() + d.Session.DB(DB_NAME).C(USERS_C).DropCollection() + + date1, _ := time.Parse(SHORT_DATE_LAYOUT, "1994-Jul-05") + date2, _ := time.Parse(SHORT_DATE_LAYOUT, "1998-Sep-04") + date3, _ := time.Parse(SHORT_DATE_LAYOUT, "2008-Apr-28") + + migrations := []struct { + name string + file file.File + expectedResult ExpectedMigrationResult + }{ + { + name: "v0 -> v1", + file: file.File{ + Path: "/foobar", + FileName: "001_foobar.up.gm", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V001_init_organizations_up + V001_init_users_up + `), + }, + expectedResult: ExpectedMigrationResult{ + Organizations: []Organization{ + {Id: OrganizationIds[0], Name: "Amazon", Location: "Seattle", DateFounded: date1}, + {Id: OrganizationIds[1], Name: "Google", Location: "Mountain View", DateFounded: date2}, + {Id: OrganizationIds[2], Name: "JFrog", Location: "Santa Clara", DateFounded: date3}, + }, + Organizations_v2: []Organization_v2{}, + Users: []User{ + {Id: UserIds[0], Name: "Alex"}, + {Id: UserIds[1], Name: "Beatrice"}, + {Id: UserIds[2], Name: "Cleo"}, + }, + Errors: []error{}, + }, + }, + { + name: "v1 -> v2", + file: file.File{ + Path: "/foobar", + FileName: "002_foobar.up.gm", + Version: 2, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V002_organizations_rename_location_field_to_headquarters_up + V002_change_user_cleo_to_cleopatra_up + `), + }, + expectedResult: ExpectedMigrationResult{ + Organizations: []Organization{}, + Organizations_v2: []Organization_v2{ + {Id: OrganizationIds[0], Name: "Amazon", Headquarters: "Seattle", DateFounded: date1}, + {Id: OrganizationIds[1], Name: "Google", Headquarters: "Mountain View", DateFounded: date2}, + {Id: OrganizationIds[2], Name: "JFrog", Headquarters: "Santa Clara", DateFounded: date3}, + }, + Users: []User{ + {Id: UserIds[0], Name: "Alex"}, + {Id: UserIds[1], Name: "Beatrice"}, + {Id: UserIds[2], Name: "Cleopatra"}, + }, + Errors: []error{}, + }, + }, + { + name: "v2 -> v1", + file: file.File{ + Path: "/foobar", + FileName: "002_foobar.down.gm", + Version: 2, + Name: "foobar", + Direction: direction.Down, + Content: []byte(` + V002_change_user_cleo_to_cleopatra_down + V002_organizations_rename_location_field_to_headquarters_down + `), + }, + expectedResult: ExpectedMigrationResult{ + Organizations: []Organization{ + {Id: OrganizationIds[0], Name: "Amazon", Location: "Seattle", DateFounded: date1}, + {Id: OrganizationIds[1], Name: "Google", Location: "Mountain View", DateFounded: date2}, + {Id: OrganizationIds[2], Name: "JFrog", Location: "Santa Clara", DateFounded: date3}, + }, + Organizations_v2: []Organization_v2{}, + Users: []User{ + {Id: UserIds[0], Name: "Alex"}, + {Id: UserIds[1], Name: "Beatrice"}, + {Id: UserIds[2], Name: "Cleo"}, + }, + Errors: []error{}, + }, + }, + { + name: "v1 -> v0", + file: file.File{ + Path: "/foobar", + FileName: "001_foobar.down.gm", + Version: 1, + Name: "foobar", + Direction: direction.Down, + Content: []byte(` + V001_init_users_down + V001_init_organizations_down + `), + }, + expectedResult: ExpectedMigrationResult{ + Organizations: []Organization{}, + Organizations_v2: []Organization_v2{}, + Users: []User{}, + Errors: []error{}, + }, + }, + { + name: "v0 -> v1: missing method aborts migration", + file: file.File{ + Path: "/foobar", + FileName: "001_foobar.up.gm", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V001_init_organizations_up + V001_init_users_up + v001_non_existing_method_up + `), + }, + expectedResult: ExpectedMigrationResult{ + Organizations: []Organization{}, + Organizations_v2: []Organization_v2{}, + Users: []User{}, + Errors: []error{gomethods.MethodNotFoundError("v001_non_existing_method_up")}, + }, + }, + { + name: "v0 -> v1: not exported method aborts migration", + file: file.File{ + Path: "/foobar", + FileName: "001_foobar.up.gm", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V001_init_organizations_up + v001_not_exported_method_up + V001_init_users_up + `), + }, + expectedResult: ExpectedMigrationResult{ + Organizations: []Organization{}, + Organizations_v2: []Organization_v2{}, + Users: []User{}, + Errors: []error{gomethods.MethodNotFoundError("v001_not_exported_method_up")}, + }, + }, + { + name: "v0 -> v1: wrong signature method aborts migration", + file: file.File{ + Path: "/foobar", + FileName: "001_foobar.up.gm", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V001_init_organizations_up + V001_method_with_wrong_signature_up + V001_init_users_up + `), + }, + expectedResult: ExpectedMigrationResult{ + Organizations: []Organization{}, + Organizations_v2: []Organization_v2{}, + Users: []User{}, + Errors: []error{gomethods.WrongMethodSignatureError("V001_method_with_wrong_signature_up")}, + }, + }, + { + name: "v1 -> v0: wrong signature method aborts migration", + file: file.File{ + Path: "/foobar", + FileName: "001_foobar.down.gm", + Version: 1, + Name: "foobar", + Direction: direction.Down, + Content: []byte(` + V001_init_users_down + V001_method_with_wrong_signature_down + V001_init_organizations_down + `), + }, + expectedResult: ExpectedMigrationResult{ + Organizations: []Organization{}, + Organizations_v2: []Organization_v2{}, + Users: []User{}, + Errors: []error{gomethods.WrongMethodSignatureError("V001_method_with_wrong_signature_down")}, + }, + }, + } + + for _, m := range migrations { + RunMigrationAndAssertResult(t, m.name, d, m.file, &m.expectedResult) + } + + if err := d.Close(); err != nil { + t.Fatal(err) + } +} diff --git a/driver/mongodb/example/sample_mongdb_migrator.go b/driver/mongodb/example/sample_mongdb_migrator.go new file mode 100644 index 00000000..1ab1440a --- /dev/null +++ b/driver/mongodb/example/sample_mongdb_migrator.go @@ -0,0 +1,153 @@ +package example + +import ( + "github.com/mattes/migrate/driver/mongodb/gomethods" + _ "github.com/mattes/migrate/driver/mongodb/gomethods" + "gopkg.in/mgo.v2" + "gopkg.in/mgo.v2/bson" + "time" + + "github.com/mattes/migrate/driver/mongodb" +) + +type SampleMongoDbMigrator struct { +} + +func (r *SampleMongoDbMigrator) DbName() string { + return DB_NAME +} + +var _ mongodb.MethodsReceiver = (*SampleMongoDbMigrator)(nil) + +func init() { + gomethods.RegisterMethodsReceiverForDriver("mongodb", &SampleMongoDbMigrator{}) +} + +// Here goes the specific mongodb golang methods migration logic + +const ( + DB_NAME = "test" + SHORT_DATE_LAYOUT = "2000-Jan-01" + USERS_C = "users" + ORGANIZATIONS_C = "organizations" +) + +type Organization struct { + Id bson.ObjectId `bson:"_id,omitempty"` + Name string `bson:"name"` + Location string `bson:"location"` + DateFounded time.Time `bson:"date_founded"` +} + +type Organization_v2 struct { + Id bson.ObjectId `bson:"_id,omitempty"` + Name string `bson:"name"` + Headquarters string `bson:"headquarters"` + DateFounded time.Time `bson:"date_founded"` +} + +type User struct { + Id bson.ObjectId `bson:"_id"` + Name string `bson:"name"` +} + +var OrganizationIds []bson.ObjectId = []bson.ObjectId{ + bson.NewObjectId(), + bson.NewObjectId(), + bson.NewObjectId(), +} + +var UserIds []bson.ObjectId = []bson.ObjectId{ + bson.NewObjectId(), + bson.NewObjectId(), + bson.NewObjectId(), +} + +func (r *SampleMongoDbMigrator) V001_init_organizations_up(session *mgo.Session) error { + date1, _ := time.Parse(SHORT_DATE_LAYOUT, "1994-Jul-05") + date2, _ := time.Parse(SHORT_DATE_LAYOUT, "1998-Sep-04") + date3, _ := time.Parse(SHORT_DATE_LAYOUT, "2008-Apr-28") + + orgs := []Organization{ + {Id: OrganizationIds[0], Name: "Amazon", Location: "Seattle", DateFounded: date1}, + {Id: OrganizationIds[1], Name: "Google", Location: "Mountain View", DateFounded: date2}, + {Id: OrganizationIds[2], Name: "JFrog", Location: "Santa Clara", DateFounded: date3}, + } + + for _, org := range orgs { + err := session.DB(DB_NAME).C(ORGANIZATIONS_C).Insert(org) + if err != nil { + return err + } + } + return nil +} + +func (r *SampleMongoDbMigrator) V001_init_organizations_down(session *mgo.Session) error { + return session.DB(DB_NAME).C(ORGANIZATIONS_C).DropCollection() +} + +func (r *SampleMongoDbMigrator) V001_init_users_up(session *mgo.Session) error { + users := []User{ + {Id: UserIds[0], Name: "Alex"}, + {Id: UserIds[1], Name: "Beatrice"}, + {Id: UserIds[2], Name: "Cleo"}, + } + + for _, user := range users { + err := session.DB(DB_NAME).C(USERS_C).Insert(user) + if err != nil { + return err + } + } + return nil +} + +func (r *SampleMongoDbMigrator) V001_init_users_down(session *mgo.Session) error { + return session.DB(DB_NAME).C(USERS_C).DropCollection() +} + +func (r *SampleMongoDbMigrator) V002_organizations_rename_location_field_to_headquarters_up(session *mgo.Session) error { + c := session.DB(DB_NAME).C(ORGANIZATIONS_C) + + _, err := c.UpdateAll(nil, bson.M{"$rename": bson.M{"location": "headquarters"}}) + return err +} + +func (r *SampleMongoDbMigrator) V002_organizations_rename_location_field_to_headquarters_down(session *mgo.Session) error { + c := session.DB(DB_NAME).C(ORGANIZATIONS_C) + + _, err := c.UpdateAll(nil, bson.M{"$rename": bson.M{"headquarters": "location"}}) + return err +} + +func (r *SampleMongoDbMigrator) V002_change_user_cleo_to_cleopatra_up(session *mgo.Session) error { + c := session.DB(DB_NAME).C(USERS_C) + + colQuerier := bson.M{"name": "Cleo"} + change := bson.M{"$set": bson.M{"name": "Cleopatra"}} + + return c.Update(colQuerier, change) +} + +func (r *SampleMongoDbMigrator) V002_change_user_cleo_to_cleopatra_down(session *mgo.Session) error { + c := session.DB(DB_NAME).C(USERS_C) + + colQuerier := bson.M{"name": "Cleopatra"} + change := bson.M{"$set": bson.M{"name": "Cleo"}} + + return c.Update(colQuerier, change) +} + +// Wrong signature methods for testing +func (r *SampleMongoDbMigrator) v001_not_exported_method_up(session *mgo.Session) error { + return nil +} + +func (r *SampleMongoDbMigrator) V001_method_with_wrong_signature_up(s string) error { + return nil +} + +func (r *SampleMongoDbMigrator) V001_method_with_wrong_signature_down(session *mgo.Session) (bool, error) { + return true, nil +} diff --git a/driver/mongodb/gomethods/gomethods_migrator.go b/driver/mongodb/gomethods/gomethods_migrator.go new file mode 100644 index 00000000..797c6f29 --- /dev/null +++ b/driver/mongodb/gomethods/gomethods_migrator.go @@ -0,0 +1,149 @@ +package gomethods + +import ( + "bufio" + "fmt" + "github.com/mattes/migrate/driver" + "github.com/mattes/migrate/file" + "os" + "path" + "strings" +) + +type WrongMethodSignatureError string + +func (e WrongMethodSignatureError) Error() string { + return fmt.Sprintf("Method '%s' has wrong signature", string(e)) +} + +type MethodNotFoundError string + +func (e MethodNotFoundError) Error() string { + return fmt.Sprintf("Method '%s' was not found. It is either non-existing or has not been exported (starts with lowercase).", string(e)) +} + +type MethodInvocationFailedError struct { + MethodName string + Err error +} + +func (e *MethodInvocationFailedError) Error() string { + return fmt.Sprintf("Method '%s' returned an error: %v", e.MethodName, e.Err) +} + +type MigrationMethodInvoker interface { + Validate(methodName string) error + Invoke(methodName string) error +} + +type GoMethodsDriver interface { + driver.Driver + + MigrationMethodInvoker + MethodsReceiver() interface{} + SetMethodsReceiver(r interface{}) error +} + +type Migrator struct { + RollbackOnFailure bool + MethodInvoker MigrationMethodInvoker +} + +func (m *Migrator) Migrate(f file.File, pipe chan interface{}) error { + methods, err := m.getMigrationMethods(f) + if err != nil { + pipe <- err + return err + } + + for i, methodName := range methods { + pipe <- methodName + err := m.MethodInvoker.Invoke(methodName) + if err != nil { + pipe <- err + if !m.RollbackOnFailure { + return err + } + + // on failure, try to rollback methods in this migration + for j := i - 1; j >= 0; j-- { + rollbackToMethodName := getRollbackToMethod(methods[j]) + if rollbackToMethodName == "" { + continue + } + if err := m.MethodInvoker.Validate(rollbackToMethodName); err != nil { + continue + } + + pipe <- rollbackToMethodName + err = m.MethodInvoker.Invoke(rollbackToMethodName) + if err != nil { + pipe <- err + break + } + } + return err + } + } + + return nil +} + +func getRollbackToMethod(methodName string) string { + if strings.HasSuffix(methodName, "_up") { + return strings.TrimSuffix(methodName, "_up") + "_down" + } else if strings.HasSuffix(methodName, "_down") { + return strings.TrimSuffix(methodName, "_down") + "_up" + } else { + return "" + } +} + +func getFileLines(file file.File) ([]string, error) { + if len(file.Content) == 0 { + lines := make([]string, 0) + file, err := os.Open(path.Join(file.Path, file.FileName)) + if err != nil { + return nil, err + } + defer file.Close() + + scanner := bufio.NewScanner(file) + + for scanner.Scan() { + lines = append(lines, scanner.Text()) + } + return lines, nil + } else { + s := string(file.Content) + return strings.Split(s, "\n"), nil + } +} + +func (m *Migrator) getMigrationMethods(f file.File) (methods []string, err error) { + var lines []string + + lines, err = getFileLines(f) + if err != nil { + return nil, err + } + + for _, line := range lines { + line := strings.TrimSpace(line) + + if line == "" || strings.HasPrefix(line, "--") { + // an empty line or a comment, ignore + continue + } + + methodName := line + if err := m.MethodInvoker.Validate(methodName); err != nil { + return nil, err + } + + methods = append(methods, methodName) + } + + return methods, nil + +} diff --git a/driver/mongodb/gomethods/gomethods_migrator_test.go b/driver/mongodb/gomethods/gomethods_migrator_test.go new file mode 100644 index 00000000..d94506ee --- /dev/null +++ b/driver/mongodb/gomethods/gomethods_migrator_test.go @@ -0,0 +1,247 @@ +package gomethods + +import ( + "reflect" + "testing" + + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" + + pipep "github.com/mattes/migrate/pipe" +) + +type FakeGoMethodsInvoker struct { + InvokedMethods []string +} + +func (invoker *FakeGoMethodsInvoker) Validate(methodName string) error { + if methodName == "V001_some_non_existing_method_up" { + return MethodNotFoundError(methodName) + } + + return nil +} + +func (invoker *FakeGoMethodsInvoker) Invoke(methodName string) error { + invoker.InvokedMethods = append(invoker.InvokedMethods, methodName) + + if methodName == "V001_some_failing_method_up" || methodName == "V001_some_failing_method_down" { + return &MethodInvocationFailedError{ + MethodName: methodName, + Err: SomeError{}, + } + } + return nil +} + +type SomeError struct{} + +func (e SomeError) Error() string { return "Some error happened" } + +func TestMigrate(t *testing.T) { + cases := []struct { + name string + file file.File + expectedInvokedMethods []string + expectedErrors []error + expectRollback bool + }{ + { + name: "up migration invokes up methods", + file: file.File{ + Path: "/foobar", + FileName: "001_foobar.up.gm", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V001_init_organizations_up + V001_init_users_up + `), + }, + expectedInvokedMethods: []string{"V001_init_organizations_up", "V001_init_users_up"}, + expectedErrors: []error{}, + }, + { + name: "down migration invoked down methods", + file: file.File{ + Path: "/foobar", + FileName: "001_foobar.down.gm", + Version: 1, + Name: "foobar", + Direction: direction.Down, + Content: []byte(` + V001_init_users_down + V001_init_organizations_down + `), + }, + expectedInvokedMethods: []string{"V001_init_users_down", "V001_init_organizations_down"}, + expectedErrors: []error{}, + }, + { + name: "up migration: non-existing method causes migration not to execute", + file: file.File{ + Path: "/foobar", + FileName: "001_foobar.up.gm", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V001_init_organizations_up + V001_init_users_up + V001_some_non_existing_method_up + `), + }, + expectedInvokedMethods: []string{}, + expectedErrors: []error{MethodNotFoundError("V001_some_non_existing_method_up")}, + }, + { + name: "up migration: failing method stops execution", + file: file.File{ + Path: "/foobar", + FileName: "001_foobar.up.gm", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V001_init_organizations_up + V001_some_failing_method_up + V001_init_users_up + `), + }, + expectedInvokedMethods: []string{ + "V001_init_organizations_up", + "V001_some_failing_method_up", + }, + expectedErrors: []error{&MethodInvocationFailedError{ + MethodName: "V001_some_failing_method_up", + Err: SomeError{}, + }}, + }, + { + name: "down migration: failing method stops migration", + file: file.File{ + Path: "/foobar", + FileName: "001_foobar.down.gm", + Version: 1, + Name: "foobar", + Direction: direction.Down, + Content: []byte(` + V001_init_users_down + V001_some_failing_method_down + V001_init_organizations_down + `), + }, + expectedInvokedMethods: []string{ + "V001_init_users_down", + "V001_some_failing_method_down", + }, + expectedErrors: []error{&MethodInvocationFailedError{ + MethodName: "V001_some_failing_method_down", + Err: SomeError{}, + }}, + }, + { + name: "up migration: failing method causes rollback in rollback mode", + expectRollback: true, + file: file.File{ + Path: "/foobar", + FileName: "001_foobar.up.gm", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V001_init_organizations_up + V001_init_users_up + V001_some_failing_method_up + `), + }, + expectedInvokedMethods: []string{ + "V001_init_organizations_up", + "V001_init_users_up", + "V001_some_failing_method_up", + "V001_init_users_down", + "V001_init_organizations_down", + }, + expectedErrors: []error{&MethodInvocationFailedError{ + MethodName: "V001_some_failing_method_up", + Err: SomeError{}, + }}, + }, + { + name: "down migration: failing method causes rollback in rollback mode", + expectRollback: true, + file: file.File{ + Path: "/foobar", + FileName: "001_foobar.down.gm", + Version: 1, + Name: "foobar", + Direction: direction.Down, + Content: []byte(` + V001_init_users_down + V001_some_failing_method_down + V001_init_organizations_down + `), + }, + expectedInvokedMethods: []string{ + "V001_init_users_down", + "V001_some_failing_method_down", + "V001_init_users_up", + }, + expectedErrors: []error{&MethodInvocationFailedError{ + MethodName: "V001_some_failing_method_down", + Err: SomeError{}, + }}, + }, + } + + for _, c := range cases { + migrator := Migrator{} + fakeInvoker := &FakeGoMethodsInvoker{InvokedMethods: []string{}} + + migrator.MethodInvoker = fakeInvoker + migrator.RollbackOnFailure = c.expectRollback + + pipe := pipep.New() + go func() { + migrator.Migrate(c.file, pipe) + close(pipe) + }() + errs := pipep.ReadErrors(pipe) + + var failed bool + if !reflect.DeepEqual(fakeInvoker.InvokedMethods, c.expectedInvokedMethods) { + failed = true + t.Errorf("case '%s': FAILED\nexpected invoked methods %v\nbut got %v", c.name, c.expectedInvokedMethods, fakeInvoker.InvokedMethods) + } + if !reflect.DeepEqual(errs, c.expectedErrors) { + failed = true + t.Errorf("case '%s': FAILED\nexpected errors %v\nbut got %v", c.name, c.expectedErrors, errs) + + } + if !failed { + //t.Logf("case '%s': PASSED", c.name) + } + } +} + +func TestGetRollbackToMethod(t *testing.T) { + cases := []struct { + method string + expectedRollbackMethod string + }{ + {"some_method_up", "some_method_down"}, + {"some_method_down", "some_method_up"}, + {"up_down_up", "up_down_down"}, + {"down_up", "down_down"}, + {"down_down", "down_up"}, + {"some_method", ""}, + } + + for _, c := range cases { + actualRollbackMethod := getRollbackToMethod(c.method) + if actualRollbackMethod != c.expectedRollbackMethod { + t.Errorf("Expected rollback method to be %s but got %s", c.expectedRollbackMethod, actualRollbackMethod) + } + } +} diff --git a/driver/mongodb/gomethods/gomethods_registry.go b/driver/mongodb/gomethods/gomethods_registry.go new file mode 100644 index 00000000..418256fd --- /dev/null +++ b/driver/mongodb/gomethods/gomethods_registry.go @@ -0,0 +1,39 @@ +package gomethods + +import ( + "fmt" + "github.com/mattes/migrate/driver" + "sync" +) + +var methodsReceiversMu sync.Mutex + +// Registers a methods receiver for go methods driver +// Users of gomethods migration drivers should call this method +// to register objects with their migration methods before executing the migration +func RegisterMethodsReceiverForDriver(driverName string, receiver interface{}) { + methodsReceiversMu.Lock() + defer methodsReceiversMu.Unlock() + if receiver == nil { + panic("Go methods: Register receiver object is nil") + } + + driver := driver.GetDriver(driverName) + if driver == nil { + panic("Go methods: Trying to register receiver for not registered driver " + driverName) + } + + methodsDriver, ok := driver.(GoMethodsDriver) + if !ok { + panic("Go methods: Trying to register receiver for non go methods driver " + driverName) + } + + if methodsDriver.MethodsReceiver() != nil { + panic("Go methods: Methods receiver already registered for driver " + driverName) + } + + if err := methodsDriver.SetMethodsReceiver(receiver); err != nil { + panic(fmt.Sprintf("Go methods: Failed to set methods receiver for driver %s\nError: %v", + driverName, err)) + } +} diff --git a/driver/mongodb/mongodb.go b/driver/mongodb/mongodb.go new file mode 100644 index 00000000..a9fe9a63 --- /dev/null +++ b/driver/mongodb/mongodb.go @@ -0,0 +1,193 @@ +package mongodb + +import ( + "errors" + "github.com/mattes/migrate/driver" + "github.com/mattes/migrate/driver/mongodb/gomethods" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" + "gopkg.in/mgo.v2" + "gopkg.in/mgo.v2/bson" + "reflect" + "strings" +) + +type UnregisteredMethodsReceiverError string + +func (e UnregisteredMethodsReceiverError) Error() string { + return "Unregistered methods receiver for driver: " + string(e) +} + +type WrongMethodsReceiverTypeError string + +func (e WrongMethodsReceiverTypeError) Error() string { + return "Wrong methods receiver type for driver: " + string(e) +} + +const MIGRATE_C = "db_migrations" +const DRIVER_NAME = "gomethods.mongodb" + +type Driver struct { + Session *mgo.Session + + methodsReceiver MethodsReceiver + migrator gomethods.Migrator +} + +var _ gomethods.GoMethodsDriver = (*Driver)(nil) + +type MethodsReceiver interface { + DbName() string +} + +func (d *Driver) MethodsReceiver() interface{} { + return d.methodsReceiver +} + +func (d *Driver) SetMethodsReceiver(r interface{}) error { + r1, ok := r.(MethodsReceiver) + if !ok { + return WrongMethodsReceiverTypeError(DRIVER_NAME) + } + + d.methodsReceiver = r1 + return nil +} + +func init() { + driver.RegisterDriver("mongodb", &Driver{}) +} + +type DbMigration struct { + Id bson.ObjectId `bson:"_id,omitempty"` + Version uint64 `bson:"version"` +} + +func (driver *Driver) Initialize(url string) error { + if driver.methodsReceiver == nil { + return UnregisteredMethodsReceiverError(DRIVER_NAME) + } + + urlWithoutScheme := strings.SplitN(url, "mongodb://", 2) + if len(urlWithoutScheme) != 2 { + return errors.New("invalid mongodb:// scheme") + } + + session, err := mgo.Dial(url) + if err != nil { + return err + } + session.SetMode(mgo.Monotonic, true) + + c := session.DB(driver.methodsReceiver.DbName()).C(MIGRATE_C) + err = c.EnsureIndex(mgo.Index{ + Key: []string{"version"}, + Unique: true, + }) + if err != nil { + return err + } + + driver.Session = session + driver.migrator = gomethods.Migrator{MethodInvoker: driver} + + return nil +} + +func (driver *Driver) Close() error { + if driver.Session != nil { + driver.Session.Close() + } + return nil +} + +func (driver *Driver) FilenameExtension() string { + return "mgo" +} + +func (driver *Driver) Version() (uint64, error) { + var latestMigration DbMigration + c := driver.Session.DB(driver.methodsReceiver.DbName()).C(MIGRATE_C) + + err := c.Find(bson.M{}).Sort("-version").One(&latestMigration) + + switch { + case err == mgo.ErrNotFound: + return 0, nil + case err != nil: + return 0, err + default: + return latestMigration.Version, nil + } +} +func (driver *Driver) Migrate(f file.File, pipe chan interface{}) { + defer close(pipe) + pipe <- f + + err := driver.migrator.Migrate(f, pipe) + if err != nil { + return + } + + migrate_c := driver.Session.DB(driver.methodsReceiver.DbName()).C(MIGRATE_C) + + if f.Direction == direction.Up { + id := bson.NewObjectId() + dbMigration := DbMigration{Id: id, Version: f.Version} + + err := migrate_c.Insert(dbMigration) + if err != nil { + pipe <- err + return + } + + } else if f.Direction == direction.Down { + err := migrate_c.Remove(bson.M{"version": f.Version}) + if err != nil { + pipe <- err + return + } + } +} + +func (driver *Driver) Validate(methodName string) error { + methodWithReceiver, ok := reflect.TypeOf(driver.methodsReceiver).MethodByName(methodName) + if !ok { + return gomethods.MethodNotFoundError(methodName) + } + if methodWithReceiver.PkgPath != "" { + return gomethods.MethodNotFoundError(methodName) + } + + methodFunc := reflect.ValueOf(driver.methodsReceiver).MethodByName(methodName) + methodTemplate := func(*mgo.Session) error { return nil } + + if methodFunc.Type() != reflect.TypeOf(methodTemplate) { + return gomethods.WrongMethodSignatureError(methodName) + } + + return nil +} + +func (driver *Driver) Invoke(methodName string) error { + name := methodName + migrateMethod := reflect.ValueOf(driver.methodsReceiver).MethodByName(name) + if !migrateMethod.IsValid() { + return gomethods.MethodNotFoundError(methodName) + } + + retValues := migrateMethod.Call([]reflect.Value{reflect.ValueOf(driver.Session)}) + if len(retValues) != 1 { + return gomethods.WrongMethodSignatureError(name) + } + + if !retValues[0].IsNil() { + err, ok := retValues[0].Interface().(error) + if !ok { + return gomethods.WrongMethodSignatureError(name) + } + return &gomethods.MethodInvocationFailedError{MethodName: name, Err: err} + } + + return nil +} diff --git a/driver/mysql/mysql_test.go b/driver/mysql/mysql_test.go index 51cf552a..ec192716 100644 --- a/driver/mysql/mysql_test.go +++ b/driver/mysql/mysql_test.go @@ -14,6 +14,9 @@ import ( // TestMigrate runs some additional tests on Migrate(). // Basic testing is already done in migrate/migrate_test.go func TestMigrate(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } host := os.Getenv("MYSQL_PORT_3306_TCP_ADDR") port := os.Getenv("MYSQL_PORT_3306_TCP_PORT") driverUrl := "mysql://root@tcp(" + host + ":" + port + ")/migratetest" diff --git a/driver/postgres/README.md b/driver/postgres/README.md index b7d12b74..213e4a78 100644 --- a/driver/postgres/README.md +++ b/driver/postgres/README.md @@ -1,7 +1,7 @@ # PostgreSQL Driver -* Runs migrations in transcations. - That means that if a migration failes, it will be safely rolled back. +* Runs migrations in transactions. + That means that if a migration fails, it will be safely rolled back. * Tries to return helpful error messages. * Stores migration version details in table ``schema_migrations``. This table will be auto-generated. @@ -15,9 +15,11 @@ migrate -url postgres://user@host:port/database -path ./db/migrations up migrate help # for more info # TODO(mattes): thinking about adding some custom flag to allow migration within schemas: --url="postgres://user@host:port/database?schema=name" +-url="postgres://user@host:port/database?schema=name" + +# see more docs: https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parameters ``` ## Authors -* Matthias Kadenbach, https://github.com/mattes \ No newline at end of file +* Matthias Kadenbach, https://github.com/mattes diff --git a/driver/postgres/postgres_test.go b/driver/postgres/postgres_test.go index ec91b350..a1c04958 100644 --- a/driver/postgres/postgres_test.go +++ b/driver/postgres/postgres_test.go @@ -13,6 +13,10 @@ import ( // TestMigrate runs some additional tests on Migrate(). // Basic testing is already done in migrate/migrate_test.go func TestMigrate(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } + host := os.Getenv("POSTGRES_PORT_5432_TCP_ADDR") port := os.Getenv("POSTGRES_PORT_5432_TCP_PORT") driverUrl := "postgres://postgres@" + host + ":" + port + "/template1?sslmode=disable" diff --git a/driver/ql/README.md b/driver/ql/README.md new file mode 100644 index 00000000..de53f814 --- /dev/null +++ b/driver/ql/README.md @@ -0,0 +1,17 @@ +# Ql Driver + +* Supports both in-memory and file ql databases, with the URL schemes `ql+file://` and `ql+memory://` + * In-memory driver is not of much use, since the database is destroyed once the migration operation finishes, but is included for completeness. +* Stores migration version details in automatically generated table ``schema_migrations``. + +## Usage + +```bash +migrate -url ql+file://./data.db -path ./migrations create add_field_to_table +migrate -url ql+file://./data.db -path ./db/migrations up +migrate help # for more info +``` + +## Authors + +* Sam Willcocks, https://github.com/wlcx diff --git a/driver/ql/ql.go b/driver/ql/ql.go new file mode 100644 index 00000000..1c75fe70 --- /dev/null +++ b/driver/ql/ql.go @@ -0,0 +1,130 @@ +package ql + +import ( + "database/sql" + + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/driver" + "github.com/mattes/migrate/migrate/direction" + + _ "github.com/cznic/ql/driver" + "strings" + "fmt" +) + +const tableName = "schema_migration" + +type Driver struct { + db *sql.DB +} + +func (d *Driver) Initialize(url string) (err error) { + d.db, err = sql.Open("ql", strings.TrimPrefix(url, "ql+")) + if err != nil { + return + } + if err = d.db.Ping(); err != nil { + return + } + if err = d.ensureVersionTableExists(); err != nil { + return + } + return +} + +func (d *Driver) Close() error { + if err := d.db.Close(); err != nil { + return err + } + return nil +} + +func (d *Driver) FilenameExtension() string { + return "sql" +} + +func (d *Driver) Migrate(f file.File, pipe chan interface{}) { + defer close(pipe) + pipe <- f + + tx, err := d.db.Begin() + if err != nil { + pipe <- err + return + } + + switch f.Direction { + case direction.Up: + if _, err := tx.Exec("INSERT INTO "+tableName+" (version) VALUES (uint($1))", f.Version); err != nil { + pipe <- err + if err := tx.Rollback(); err != nil { + pipe <- err + } + return + } + case direction.Down: + if _, err := tx.Exec("DELETE FROM "+tableName+" WHERE version=uint($1)", f.Version); err != nil { + pipe <- err + if err := tx.Rollback(); err != nil { + pipe <- err + } + return + } + } + + if err := f.ReadContent(); err != nil { + pipe <- err + return + } + + if _, err := tx.Exec(string(f.Content)); err != nil { + pipe <- err + if err := tx.Rollback(); err != nil { + pipe <- err + } + return + } + + if err := tx.Commit(); err != nil { + pipe <- err + return + } +} + +func (d *Driver) Version() (uint64, error) { + var version uint64 + err := d.db.QueryRow("SELECT version FROM " + tableName + " ORDER BY version DESC LIMIT 1").Scan(&version) + switch { + case err == sql.ErrNoRows: + return 0, nil + case err != nil: + return 0, err + default: + return version, nil + } +} + +func (d *Driver) ensureVersionTableExists() error { + tx, err := d.db.Begin() + if err != nil { + return err + } + if _, err := tx.Exec(fmt.Sprintf(` + CREATE TABLE IF NOT EXISTS %s (version uint64); + CREATE UNIQUE INDEX IF NOT EXISTS version_unique ON %s (version); + `, tableName, tableName)); err != nil { + if err := tx.Rollback(); err != nil { + return err + } + return err + } + if err := tx.Commit(); err != nil { + return err + } + return nil +} + +func init() { + driver.RegisterDriver("ql+file", &Driver{}) + driver.RegisterDriver("ql+memory", &Driver{}) +} \ No newline at end of file diff --git a/driver/registry.go b/driver/registry.go index 86a3955b..63889184 100644 --- a/driver/registry.go +++ b/driver/registry.go @@ -9,7 +9,7 @@ var driversMu sync.Mutex var drivers = make(map[string]Driver) // Registers a driver so it can be created from its name. Drivers should -// call this from an init() function so that they registers themselvse on +// call this from an init() function so that they registers themselves on // import func RegisterDriver(name string, driver Driver) { driversMu.Lock() diff --git a/driver/sqlite3/sqlite3_test.go b/driver/sqlite3/sqlite3_test.go index 8d51a7de..7f50095a 100644 --- a/driver/sqlite3/sqlite3_test.go +++ b/driver/sqlite3/sqlite3_test.go @@ -12,6 +12,10 @@ import ( // TestMigrate runs some additional tests on Migrate() // Basic testing is already done in migrate/migrate_test.go func TestMigrate(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } + driverFile := ":memory:" driverUrl := "sqlite3://" + driverFile diff --git a/main.go b/main.go index b6da438f..066b102b 100644 --- a/main.go +++ b/main.go @@ -13,8 +13,10 @@ import ( "github.com/fatih/color" _ "github.com/mattes/migrate/driver/bash" _ "github.com/mattes/migrate/driver/cassandra" + _ "github.com/mattes/migrate/driver/crate" _ "github.com/mattes/migrate/driver/mysql" _ "github.com/mattes/migrate/driver/postgres" + _ "github.com/mattes/migrate/driver/ql" _ "github.com/mattes/migrate/driver/sqlite3" "github.com/mattes/migrate/file" "github.com/mattes/migrate/migrate" @@ -158,7 +160,8 @@ func main() { fmt.Println(version) default: - fallthrough + helpCmd() + os.Exit(1) case "help": helpCmd() } diff --git a/migrate/migrate.go b/migrate/migrate.go index b452aeef..0c976be9 100644 --- a/migrate/migrate.go +++ b/migrate/migrate.go @@ -10,6 +10,7 @@ import ( "path" "strconv" "strings" + "time" "github.com/mattes/migrate/driver" "github.com/mattes/migrate/file" @@ -34,11 +35,14 @@ func Up(pipe chan interface{}, url, migrationsPath string) { return } + signals := handleInterrupts() + defer signal.Stop(signals) + if len(applyMigrationFiles) > 0 { for _, f := range applyMigrationFiles { pipe1 := pipep.New() go d.Migrate(f, pipe1) - if ok := pipep.WaitAndRedirect(pipe1, pipe, handleInterrupts()); !ok { + if ok := pipep.WaitAndRedirect(pipe1, pipe, signals); !ok { break } } @@ -81,11 +85,14 @@ func Down(pipe chan interface{}, url, migrationsPath string) { return } + signals := handleInterrupts() + defer signal.Stop(signals) + if len(applyMigrationFiles) > 0 { for _, f := range applyMigrationFiles { pipe1 := pipep.New() go d.Migrate(f, pipe1) - if ok := pipep.WaitAndRedirect(pipe1, pipe, handleInterrupts()); !ok { + if ok := pipep.WaitAndRedirect(pipe1, pipe, signals); !ok { break } } @@ -115,7 +122,11 @@ func DownSync(url, migrationsPath string) (err []error, ok bool) { func Redo(pipe chan interface{}, url, migrationsPath string) { pipe1 := pipep.New() go Migrate(pipe1, url, migrationsPath, -1) - if ok := pipep.WaitAndRedirect(pipe1, pipe, handleInterrupts()); !ok { + + signals := handleInterrupts() + defer signal.Stop(signals) + + if ok := pipep.WaitAndRedirect(pipe1, pipe, signals); !ok { go pipep.Close(pipe, nil) return } else { @@ -135,7 +146,11 @@ func RedoSync(url, migrationsPath string) (err []error, ok bool) { func Reset(pipe chan interface{}, url, migrationsPath string) { pipe1 := pipep.New() go Down(pipe1, url, migrationsPath) - if ok := pipep.WaitAndRedirect(pipe1, pipe, handleInterrupts()); !ok { + + signals := handleInterrupts() + defer signal.Stop(signals) + + if ok := pipep.WaitAndRedirect(pipe1, pipe, signals); !ok { go pipep.Close(pipe, nil) return } else { @@ -168,11 +183,14 @@ func Migrate(pipe chan interface{}, url, migrationsPath string, relativeN int) { return } + signals := handleInterrupts() + defer signal.Stop(signals) + if len(applyMigrationFiles) > 0 && relativeN != 0 { for _, f := range applyMigrationFiles { pipe1 := pipep.New() go d.Migrate(f, pipe1) - if ok := pipep.WaitAndRedirect(pipe1, pipe, handleInterrupts()); !ok { + if ok := pipep.WaitAndRedirect(pipe1, pipe, signals); !ok { break } } @@ -212,20 +230,23 @@ func Create(url, migrationsPath, name string) (*file.MigrationFile, error) { if err != nil { return nil, err } + files, err := file.ReadMigrationFiles(migrationsPath, file.FilenameRegex(d.FilenameExtension())) if err != nil { return nil, err } - version := uint64(0) - if len(files) > 0 { - lastFile := files[len(files)-1] - version = lastFile.Version + version := uint64(time.Now().Unix()) + + for _, f := range files { + if f.Version == version { + version++ + } } - version += 1 + versionStr := strconv.FormatUint(version, 10) - length := 4 // TODO(mattes) check existing files and try to guess length + length := 10 if len(versionStr)%length != 0 { versionStr = strings.Repeat("0", length-len(versionStr)%length) + versionStr } diff --git a/migrate/migrate_test.go b/migrate/migrate_test.go index 7d2a622f..e202a61e 100644 --- a/migrate/migrate_test.go +++ b/migrate/migrate_test.go @@ -3,25 +3,38 @@ package migrate import ( "io/ioutil" "os" + "regexp" "testing" + // Ensure imports for each driver we wish to test _ "github.com/mattes/migrate/driver/postgres" + _ "github.com/mattes/migrate/driver/ql" _ "github.com/mattes/migrate/driver/sqlite3" ) // Add Driver URLs here to test basic Up, Down, .. functions. var driverUrls = []string{ "postgres://postgres@" + os.Getenv("POSTGRES_PORT_5432_TCP_ADDR") + ":" + os.Getenv("POSTGRES_PORT_5432_TCP_PORT") + "/template1?sslmode=disable", + "ql+file://./test.db", +} + +func tearDown(driverUrl, tmpdir string) { + DownSync(driverUrl, tmpdir) + os.RemoveAll(tmpdir) } func TestCreate(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } for _, driverUrl := range driverUrls { t.Logf("Test driver: %s", driverUrl) tmpdir, err := ioutil.TempDir("/tmp", "migrate-test") if err != nil { t.Fatal(err) } + defer tearDown(driverUrl, tmpdir) if _, err := Create(driverUrl, tmpdir, "test_migration"); err != nil { t.Fatal(err) @@ -35,208 +48,237 @@ func TestCreate(t *testing.T) { t.Fatal(err) } if len(files) != 4 { - t.Fatal("Expected 2 new files, got", len(files)) + t.Fatal("Expected 4 new files, got", len(files)) } + fileNameRegexp := regexp.MustCompile(`^\d{10}_(.*.[up|down].sql)`) + expectFiles := []string{ - "0001_test_migration.up.sql", "0001_test_migration.down.sql", - "0002_another_migration.up.sql", "0002_another_migration.down.sql", - } - foundCounter := 0 - for _, expectFile := range expectFiles { - for _, file := range files { - if expectFile == file.Name() { - foundCounter += 1 - break + "test_migration.up.sql", "test_migration.down.sql", + "another_migration.up.sql", "another_migration.down.sql", + } + + var foundCounter int + + for _, file := range files { + if x := fileNameRegexp.FindStringSubmatch(file.Name()); len(x) != 2 { + t.Errorf("expected %v to match %v", file.Name(), fileNameRegexp) + } else { + for _, expect := range expectFiles { + if expect == x[1] { + foundCounter++ + break + } } + } } + if foundCounter != len(expectFiles) { - t.Error("not all expected files have been found") + t.Errorf("expected %v files, got %v", len(expectFiles), foundCounter) } } } func TestReset(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } for _, driverUrl := range driverUrls { t.Logf("Test driver: %s", driverUrl) - tmpdir, err := ioutil.TempDir("/", "migrate-test") + tmpdir, err := ioutil.TempDir("/tmp", "migrate-test") if err != nil { t.Fatal(err) } + defer tearDown(driverUrl, tmpdir) Create(driverUrl, tmpdir, "migration1") - Create(driverUrl, tmpdir, "migration2") - - errs, ok := ResetSync(driverUrl, tmpdir) - if !ok { - t.Fatal(errs) - } - version, err := Version(driverUrl, tmpdir) + f, err := Create(driverUrl, tmpdir, "migration2") if err != nil { t.Fatal(err) } - if version != 2 { - t.Fatalf("Expected version 2, got %v", version) + + if err, ok := ResetSync(driverUrl, tmpdir); !ok { + t.Fatal(err) + } + + if version, err := Version(driverUrl, tmpdir); err != nil { + t.Fatal(err) + } else if version != f.Version { + t.Fatalf("Expected version %v, got %v", version, f.Version) } } } func TestDown(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } for _, driverUrl := range driverUrls { t.Logf("Test driver: %s", driverUrl) tmpdir, err := ioutil.TempDir("/tmp", "migrate-test") if err != nil { t.Fatal(err) } + defer tearDown(driverUrl, tmpdir) - Create(driverUrl, tmpdir, "migration1") - Create(driverUrl, tmpdir, "migration2") + initVersion, _ := Version(driverUrl, tmpdir) - errs, ok := ResetSync(driverUrl, tmpdir) - if !ok { - t.Fatal(errs) - } - version, err := Version(driverUrl, tmpdir) - if err != nil { + firstMigration, _ := Create(driverUrl, tmpdir, "migration1") + secondMigration, _ := Create(driverUrl, tmpdir, "migration2") + + t.Logf("init %v first %v second %v", initVersion, firstMigration.Version, secondMigration.Version) + + if err, ok := ResetSync(driverUrl, tmpdir); !ok { t.Fatal(err) } - if version != 2 { - t.Fatalf("Expected version 2, got %v", version) - } - errs, ok = DownSync(driverUrl, tmpdir) - if !ok { - t.Fatal(errs) + if version, err := Version(driverUrl, tmpdir); err != nil { + t.Fatal(err) + } else if version != secondMigration.Version { + t.Fatalf("Expected version %v, got %v", version, secondMigration.Version) } - version, err = Version(driverUrl, tmpdir) - if err != nil { + + if err, ok := DownSync(driverUrl, tmpdir); !ok { t.Fatal(err) } - if version != 0 { - t.Fatalf("Expected version 0, got %v", version) + + if version, err := Version(driverUrl, tmpdir); err != nil { + t.Fatal(err) + } else if version != 0 { + t.Fatalf("Expected 0, got %v", version) } } } func TestUp(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } for _, driverUrl := range driverUrls { t.Logf("Test driver: %s", driverUrl) tmpdir, err := ioutil.TempDir("/tmp", "migrate-test") if err != nil { t.Fatal(err) } + defer tearDown(driverUrl, tmpdir) - Create(driverUrl, tmpdir, "migration1") - Create(driverUrl, tmpdir, "migration2") + initVersion, _ := Version(driverUrl, tmpdir) - errs, ok := DownSync(driverUrl, tmpdir) - if !ok { - t.Fatal(errs) - } - version, err := Version(driverUrl, tmpdir) - if err != nil { + firstMigration, _ := Create(driverUrl, tmpdir, "migration1") + secondMigration, _ := Create(driverUrl, tmpdir, "migration2") + + t.Logf("init %v first %v second %v", initVersion, firstMigration.Version, secondMigration.Version) + + if err, ok := DownSync(driverUrl, tmpdir); !ok { t.Fatal(err) } - if version != 0 { - t.Fatalf("Expected version 0, got %v", version) - } - errs, ok = UpSync(driverUrl, tmpdir) - if !ok { - t.Fatal(errs) + if version, err := Version(driverUrl, tmpdir); err != nil { + t.Fatal(err) + } else if version != initVersion { + t.Fatalf("Expected initial version %v, got %v", initVersion, version) } - version, err = Version(driverUrl, tmpdir) - if err != nil { + + if err, ok := UpSync(driverUrl, tmpdir); !ok { t.Fatal(err) } - if version != 2 { - t.Fatalf("Expected version 2, got %v", version) + + if version, err := Version(driverUrl, tmpdir); err != nil { + t.Fatal(err) + } else if version != secondMigration.Version { + t.Fatalf("Expected migrated version %v, got %v", secondMigration.Version, version) } } } func TestRedo(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } for _, driverUrl := range driverUrls { t.Logf("Test driver: %s", driverUrl) tmpdir, err := ioutil.TempDir("/tmp", "migrate-test") if err != nil { t.Fatal(err) } + defer tearDown(driverUrl, tmpdir) - Create(driverUrl, tmpdir, "migration1") - Create(driverUrl, tmpdir, "migration2") + initVersion, _ := Version(driverUrl, tmpdir) - errs, ok := ResetSync(driverUrl, tmpdir) - if !ok { - t.Fatal(errs) - } - version, err := Version(driverUrl, tmpdir) - if err != nil { + firstMigration, _ := Create(driverUrl, tmpdir, "migration1") + secondMigration, _ := Create(driverUrl, tmpdir, "migration2") + + t.Logf("init %v first %v second %v", initVersion, firstMigration.Version, secondMigration.Version) + + if err, ok := ResetSync(driverUrl, tmpdir); !ok { t.Fatal(err) } - if version != 2 { - t.Fatalf("Expected version 2, got %v", version) - } - errs, ok = RedoSync(driverUrl, tmpdir) - if !ok { - t.Fatal(errs) + if version, err := Version(driverUrl, tmpdir); err != nil { + t.Fatal(err) + } else if version != secondMigration.Version { + t.Fatalf("Expected migrated version %v, got %v", secondMigration.Version, version) } - version, err = Version(driverUrl, tmpdir) - if err != nil { + + if err, ok := RedoSync(driverUrl, tmpdir); !ok { t.Fatal(err) } - if version != 2 { - t.Fatalf("Expected version 2, got %v", version) + + if version, err := Version(driverUrl, tmpdir); err != nil { + t.Fatal(err) + } else if version != secondMigration.Version { + t.Fatalf("Expected migrated version %v, got %v", secondMigration.Version, version) } } } func TestMigrate(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } for _, driverUrl := range driverUrls { t.Logf("Test driver: %s", driverUrl) tmpdir, err := ioutil.TempDir("/tmp", "migrate-test") if err != nil { t.Fatal(err) } + defer tearDown(driverUrl, tmpdir) - Create(driverUrl, tmpdir, "migration1") - Create(driverUrl, tmpdir, "migration2") + initVersion, _ := Version(driverUrl, tmpdir) - errs, ok := ResetSync(driverUrl, tmpdir) - if !ok { - t.Fatal(errs) - } - version, err := Version(driverUrl, tmpdir) - if err != nil { + firstMigration, _ := Create(driverUrl, tmpdir, "migration1") + secondMigration, _ := Create(driverUrl, tmpdir, "migration2") + + t.Logf("init %v first %v second %v", initVersion, firstMigration.Version, secondMigration.Version) + + if err, ok := ResetSync(driverUrl, tmpdir); !ok { t.Fatal(err) } - if version != 2 { - t.Fatalf("Expected version 2, got %v", version) - } - errs, ok = MigrateSync(driverUrl, tmpdir, -2) - if !ok { - t.Fatal(errs) - } - version, err = Version(driverUrl, tmpdir) - if err != nil { + if version, err := Version(driverUrl, tmpdir); err != nil { t.Fatal(err) + } else if version != secondMigration.Version { + t.Fatalf("Expected migrated version %v, got %v", secondMigration.Version, version) } - if version != 0 { - t.Fatalf("Expected version 0, got %v", version) + + if err, ok := MigrateSync(driverUrl, tmpdir, -2); !ok { + t.Fatal(err) } - errs, ok = MigrateSync(driverUrl, tmpdir, +1) - if !ok { - t.Fatal(errs) + if version, err := Version(driverUrl, tmpdir); err != nil { + t.Fatal(err) + } else if version != 0 { + t.Fatalf("Expected 0, got %v", version) } - version, err = Version(driverUrl, tmpdir) - if err != nil { + + if err, ok := MigrateSync(driverUrl, tmpdir, +1); !ok { t.Fatal(err) } - if version != 1 { - t.Fatalf("Expected version 1, got %v", version) + + if version, err := Version(driverUrl, tmpdir); err != nil { + t.Fatal(err) + } else if version != firstMigration.Version { + t.Fatalf("Expected first version %v, got %v", firstMigration.Version, version) } } }