From ade683e8b56e173e67f1b624c18970b86733c077 Mon Sep 17 00:00:00 2001 From: Brant Fitzsimmons Date: Sun, 1 Nov 2015 13:14:23 -0500 Subject: [PATCH 01/42] Fixes typos. --- driver/postgres/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/driver/postgres/README.md b/driver/postgres/README.md index b7d12b74..f83206db 100644 --- a/driver/postgres/README.md +++ b/driver/postgres/README.md @@ -1,7 +1,7 @@ # PostgreSQL Driver -* Runs migrations in transcations. - That means that if a migration failes, it will be safely rolled back. +* Runs migrations in transactions. + That means that if a migration fails, it will be safely rolled back. * Tries to return helpful error messages. * Stores migration version details in table ``schema_migrations``. This table will be auto-generated. @@ -20,4 +20,4 @@ migrate help # for more info ## Authors -* Matthias Kadenbach, https://github.com/mattes \ No newline at end of file +* Matthias Kadenbach, https://github.com/mattes From 96947759725bcbb843e7b80a1b70dbca7e49feac Mon Sep 17 00:00:00 2001 From: David Campbell Date: Tue, 15 Dec 2015 17:57:16 -0800 Subject: [PATCH 02/42] prefix migration files with epoch Using epoch prefixed filenames, multiple developers can commit migrations while minimizing versioning conflicts. This commit remains backwards compatible with all previous migrations and requires no changes to the `schema_migrations` table. This commit also cleans up migrate-test temp directories --- migrate/migrate.go | 16 +-- migrate/migrate_test.go | 224 ++++++++++++++++++++++------------------ 2 files changed, 133 insertions(+), 107 deletions(-) diff --git a/migrate/migrate.go b/migrate/migrate.go index b452aeef..f63207f1 100644 --- a/migrate/migrate.go +++ b/migrate/migrate.go @@ -10,6 +10,7 @@ import ( "path" "strconv" "strings" + "time" "github.com/mattes/migrate/driver" "github.com/mattes/migrate/file" @@ -212,20 +213,23 @@ func Create(url, migrationsPath, name string) (*file.MigrationFile, error) { if err != nil { return nil, err } + files, err := file.ReadMigrationFiles(migrationsPath, file.FilenameRegex(d.FilenameExtension())) if err != nil { return nil, err } - version := uint64(0) - if len(files) > 0 { - lastFile := files[len(files)-1] - version = lastFile.Version + version := uint64(time.Now().Unix()) + + for _, f := range files { + if f.Version == version { + version++ + } } - version += 1 + versionStr := strconv.FormatUint(version, 10) - length := 4 // TODO(mattes) check existing files and try to guess length + length := 10 if len(versionStr)%length != 0 { versionStr = strings.Repeat("0", length-len(versionStr)%length) + versionStr } diff --git a/migrate/migrate_test.go b/migrate/migrate_test.go index 7d2a622f..8d330cb4 100644 --- a/migrate/migrate_test.go +++ b/migrate/migrate_test.go @@ -3,7 +3,9 @@ package migrate import ( "io/ioutil" "os" + "regexp" "testing" + // Ensure imports for each driver we wish to test _ "github.com/mattes/migrate/driver/postgres" @@ -15,6 +17,11 @@ var driverUrls = []string{ "postgres://postgres@" + os.Getenv("POSTGRES_PORT_5432_TCP_ADDR") + ":" + os.Getenv("POSTGRES_PORT_5432_TCP_PORT") + "/template1?sslmode=disable", } +func tearDown(driverUrl, tmpdir string) { + DownSync(driverUrl, tmpdir) + os.RemoveAll(tmpdir) +} + func TestCreate(t *testing.T) { for _, driverUrl := range driverUrls { t.Logf("Test driver: %s", driverUrl) @@ -22,6 +29,7 @@ func TestCreate(t *testing.T) { if err != nil { t.Fatal(err) } + defer tearDown(driverUrl, tmpdir) if _, err := Create(driverUrl, tmpdir, "test_migration"); err != nil { t.Fatal(err) @@ -35,23 +43,33 @@ func TestCreate(t *testing.T) { t.Fatal(err) } if len(files) != 4 { - t.Fatal("Expected 2 new files, got", len(files)) + t.Fatal("Expected 4 new files, got", len(files)) } + fileNameRegexp := regexp.MustCompile(`^\d{10}_(.*.[up|down].sql)`) + expectFiles := []string{ - "0001_test_migration.up.sql", "0001_test_migration.down.sql", - "0002_another_migration.up.sql", "0002_another_migration.down.sql", - } - foundCounter := 0 - for _, expectFile := range expectFiles { - for _, file := range files { - if expectFile == file.Name() { - foundCounter += 1 - break + "test_migration.up.sql", "test_migration.down.sql", + "another_migration.up.sql", "another_migration.down.sql", + } + + var foundCounter int + + for _, file := range files { + if x := fileNameRegexp.FindStringSubmatch(file.Name()); len(x) != 2 { + t.Errorf("expected %v to match %v", file.Name(), fileNameRegexp) + } else { + for _, expect := range expectFiles { + if expect == x[1] { + foundCounter++ + break + } } + } } + if foundCounter != len(expectFiles) { - t.Error("not all expected files have been found") + t.Errorf("expected %v files, got %v", len(expectFiles), foundCounter) } } } @@ -59,24 +77,26 @@ func TestCreate(t *testing.T) { func TestReset(t *testing.T) { for _, driverUrl := range driverUrls { t.Logf("Test driver: %s", driverUrl) - tmpdir, err := ioutil.TempDir("/", "migrate-test") + tmpdir, err := ioutil.TempDir("/tmp", "migrate-test") if err != nil { t.Fatal(err) } + defer tearDown(driverUrl, tmpdir) Create(driverUrl, tmpdir, "migration1") - Create(driverUrl, tmpdir, "migration2") - - errs, ok := ResetSync(driverUrl, tmpdir) - if !ok { - t.Fatal(errs) - } - version, err := Version(driverUrl, tmpdir) + f, err := Create(driverUrl, tmpdir, "migration2") if err != nil { t.Fatal(err) } - if version != 2 { - t.Fatalf("Expected version 2, got %v", version) + + if err, ok := ResetSync(driverUrl, tmpdir); !ok { + t.Fatal(err) + } + + if version, err := Version(driverUrl, tmpdir); err != nil { + t.Fatal(err) + } else if version != f.Version { + t.Fatalf("Expected version %v, got %v", version, f.Version) } } } @@ -88,32 +108,33 @@ func TestDown(t *testing.T) { if err != nil { t.Fatal(err) } + defer tearDown(driverUrl, tmpdir) - Create(driverUrl, tmpdir, "migration1") - Create(driverUrl, tmpdir, "migration2") + initVersion, _ := Version(driverUrl, tmpdir) - errs, ok := ResetSync(driverUrl, tmpdir) - if !ok { - t.Fatal(errs) - } - version, err := Version(driverUrl, tmpdir) - if err != nil { + firstMigration, _ := Create(driverUrl, tmpdir, "migration1") + secondMigration, _ := Create(driverUrl, tmpdir, "migration2") + + t.Logf("init %v first %v second %v", initVersion, firstMigration.Version, secondMigration.Version) + + if err, ok := ResetSync(driverUrl, tmpdir); !ok { t.Fatal(err) } - if version != 2 { - t.Fatalf("Expected version 2, got %v", version) - } - errs, ok = DownSync(driverUrl, tmpdir) - if !ok { - t.Fatal(errs) + if version, err := Version(driverUrl, tmpdir); err != nil { + t.Fatal(err) + } else if version != secondMigration.Version { + t.Fatalf("Expected version %v, got %v", version, secondMigration.Version) } - version, err = Version(driverUrl, tmpdir) - if err != nil { + + if err, ok := DownSync(driverUrl, tmpdir); !ok { t.Fatal(err) } - if version != 0 { - t.Fatalf("Expected version 0, got %v", version) + + if version, err := Version(driverUrl, tmpdir); err != nil { + t.Fatal(err) + } else if version != 0 { + t.Fatalf("Expected 0, got %v", version) } } } @@ -125,32 +146,33 @@ func TestUp(t *testing.T) { if err != nil { t.Fatal(err) } + defer tearDown(driverUrl, tmpdir) - Create(driverUrl, tmpdir, "migration1") - Create(driverUrl, tmpdir, "migration2") + initVersion, _ := Version(driverUrl, tmpdir) - errs, ok := DownSync(driverUrl, tmpdir) - if !ok { - t.Fatal(errs) - } - version, err := Version(driverUrl, tmpdir) - if err != nil { + firstMigration, _ := Create(driverUrl, tmpdir, "migration1") + secondMigration, _ := Create(driverUrl, tmpdir, "migration2") + + t.Logf("init %v first %v second %v", initVersion, firstMigration.Version, secondMigration.Version) + + if err, ok := DownSync(driverUrl, tmpdir); !ok { t.Fatal(err) } - if version != 0 { - t.Fatalf("Expected version 0, got %v", version) - } - errs, ok = UpSync(driverUrl, tmpdir) - if !ok { - t.Fatal(errs) + if version, err := Version(driverUrl, tmpdir); err != nil { + t.Fatal(err) + } else if version != initVersion { + t.Fatalf("Expected initial version %v, got %v", initVersion, version) } - version, err = Version(driverUrl, tmpdir) - if err != nil { + + if err, ok := UpSync(driverUrl, tmpdir); !ok { t.Fatal(err) } - if version != 2 { - t.Fatalf("Expected version 2, got %v", version) + + if version, err := Version(driverUrl, tmpdir); err != nil { + t.Fatal(err) + } else if version != secondMigration.Version { + t.Fatalf("Expected migrated version %v, got %v", secondMigration.Version, version) } } } @@ -162,32 +184,33 @@ func TestRedo(t *testing.T) { if err != nil { t.Fatal(err) } + defer tearDown(driverUrl, tmpdir) - Create(driverUrl, tmpdir, "migration1") - Create(driverUrl, tmpdir, "migration2") + initVersion, _ := Version(driverUrl, tmpdir) - errs, ok := ResetSync(driverUrl, tmpdir) - if !ok { - t.Fatal(errs) - } - version, err := Version(driverUrl, tmpdir) - if err != nil { + firstMigration, _ := Create(driverUrl, tmpdir, "migration1") + secondMigration, _ := Create(driverUrl, tmpdir, "migration2") + + t.Logf("init %v first %v second %v", initVersion, firstMigration.Version, secondMigration.Version) + + if err, ok := ResetSync(driverUrl, tmpdir); !ok { t.Fatal(err) } - if version != 2 { - t.Fatalf("Expected version 2, got %v", version) - } - errs, ok = RedoSync(driverUrl, tmpdir) - if !ok { - t.Fatal(errs) + if version, err := Version(driverUrl, tmpdir); err != nil { + t.Fatal(err) + } else if version != secondMigration.Version { + t.Fatalf("Expected migrated version %v, got %v", secondMigration.Version, version) } - version, err = Version(driverUrl, tmpdir) - if err != nil { + + if err, ok := RedoSync(driverUrl, tmpdir); !ok { t.Fatal(err) } - if version != 2 { - t.Fatalf("Expected version 2, got %v", version) + + if version, err := Version(driverUrl, tmpdir); err != nil { + t.Fatal(err) + } else if version != secondMigration.Version { + t.Fatalf("Expected migrated version %v, got %v", secondMigration.Version, version) } } } @@ -199,44 +222,43 @@ func TestMigrate(t *testing.T) { if err != nil { t.Fatal(err) } + defer tearDown(driverUrl, tmpdir) - Create(driverUrl, tmpdir, "migration1") - Create(driverUrl, tmpdir, "migration2") + initVersion, _ := Version(driverUrl, tmpdir) - errs, ok := ResetSync(driverUrl, tmpdir) - if !ok { - t.Fatal(errs) - } - version, err := Version(driverUrl, tmpdir) - if err != nil { + firstMigration, _ := Create(driverUrl, tmpdir, "migration1") + secondMigration, _ := Create(driverUrl, tmpdir, "migration2") + + t.Logf("init %v first %v second %v", initVersion, firstMigration.Version, secondMigration.Version) + + if err, ok := ResetSync(driverUrl, tmpdir); !ok { t.Fatal(err) } - if version != 2 { - t.Fatalf("Expected version 2, got %v", version) - } - errs, ok = MigrateSync(driverUrl, tmpdir, -2) - if !ok { - t.Fatal(errs) - } - version, err = Version(driverUrl, tmpdir) - if err != nil { + if version, err := Version(driverUrl, tmpdir); err != nil { t.Fatal(err) + } else if version != secondMigration.Version { + t.Fatalf("Expected migrated version %v, got %v", secondMigration.Version, version) } - if version != 0 { - t.Fatalf("Expected version 0, got %v", version) + + if err, ok := MigrateSync(driverUrl, tmpdir, -2); !ok { + t.Fatal(err) } - errs, ok = MigrateSync(driverUrl, tmpdir, +1) - if !ok { - t.Fatal(errs) + if version, err := Version(driverUrl, tmpdir); err != nil { + t.Fatal(err) + } else if version != 0 { + t.Fatalf("Expected 0, got %v", version) } - version, err = Version(driverUrl, tmpdir) - if err != nil { + + if err, ok := MigrateSync(driverUrl, tmpdir, +1); !ok { t.Fatal(err) } - if version != 1 { - t.Fatalf("Expected version 1, got %v", version) + + if version, err := Version(driverUrl, tmpdir); err != nil { + t.Fatal(err) + } else if version != firstMigration.Version { + t.Fatalf("Expected first version %v, got %v", firstMigration.Version, version) } } } From 10d89631a3b19704e9aea8d4d048bf18193ed126 Mon Sep 17 00:00:00 2001 From: Chris McNabb Date: Sun, 10 Jan 2016 08:09:34 -0600 Subject: [PATCH 03/42] https://github.com/mattn/go-sqlite3 is cgo, so CGO_ENABLED must be true --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 507a8560..8aa43481 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,7 +14,7 @@ go-build: <<: *go command: sh -c 'go get -v && go build -ldflags ''-s'' -o migrater' environment: - CGO_ENABLED: 0 + CGO_ENABLED: 1 postgres: image: postgres mysql: From 90bab2409b538f3c9fe57502e5278618ced5a8d4 Mon Sep 17 00:00:00 2001 From: Chris McNabb Date: Sun, 10 Jan 2016 21:39:37 -0600 Subject: [PATCH 04/42] gocql doesn't support cassandra > 2.2 --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8aa43481..cac3abcf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,4 +23,4 @@ mysql: MYSQL_DATABASE: migratetest MYSQL_ALLOW_EMPTY_PASSWORD: yes cassandra: - image: cassandra + image: cassandra:2.2 From 7f9f7e59786e8ab603a01f25e511a748b041e3e8 Mon Sep 17 00:00:00 2001 From: Michael Morin Date: Thu, 11 Feb 2016 22:35:49 -0500 Subject: [PATCH 05/42] Release OS signals when you're finished with them. --- migrate/migrate.go | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/migrate/migrate.go b/migrate/migrate.go index b452aeef..0213c433 100644 --- a/migrate/migrate.go +++ b/migrate/migrate.go @@ -34,11 +34,14 @@ func Up(pipe chan interface{}, url, migrationsPath string) { return } + signals := handleInterrupts() + defer signal.Stop(signals) + if len(applyMigrationFiles) > 0 { for _, f := range applyMigrationFiles { pipe1 := pipep.New() go d.Migrate(f, pipe1) - if ok := pipep.WaitAndRedirect(pipe1, pipe, handleInterrupts()); !ok { + if ok := pipep.WaitAndRedirect(pipe1, pipe, signals); !ok { break } } @@ -81,11 +84,14 @@ func Down(pipe chan interface{}, url, migrationsPath string) { return } + signals := handleInterrupts() + defer signal.Stop(signals) + if len(applyMigrationFiles) > 0 { for _, f := range applyMigrationFiles { pipe1 := pipep.New() go d.Migrate(f, pipe1) - if ok := pipep.WaitAndRedirect(pipe1, pipe, handleInterrupts()); !ok { + if ok := pipep.WaitAndRedirect(pipe1, pipe, signals); !ok { break } } @@ -115,7 +121,11 @@ func DownSync(url, migrationsPath string) (err []error, ok bool) { func Redo(pipe chan interface{}, url, migrationsPath string) { pipe1 := pipep.New() go Migrate(pipe1, url, migrationsPath, -1) - if ok := pipep.WaitAndRedirect(pipe1, pipe, handleInterrupts()); !ok { + + signals := handleInterrupts() + defer signal.Stop(signals) + + if ok := pipep.WaitAndRedirect(pipe1, pipe, signals); !ok { go pipep.Close(pipe, nil) return } else { @@ -135,7 +145,11 @@ func RedoSync(url, migrationsPath string) (err []error, ok bool) { func Reset(pipe chan interface{}, url, migrationsPath string) { pipe1 := pipep.New() go Down(pipe1, url, migrationsPath) - if ok := pipep.WaitAndRedirect(pipe1, pipe, handleInterrupts()); !ok { + + signals := handleInterrupts() + defer signal.Stop(signals) + + if ok := pipep.WaitAndRedirect(pipe1, pipe, signals); !ok { go pipep.Close(pipe, nil) return } else { @@ -168,11 +182,14 @@ func Migrate(pipe chan interface{}, url, migrationsPath string, relativeN int) { return } + signals := handleInterrupts() + defer signal.Stop(signals) + if len(applyMigrationFiles) > 0 && relativeN != 0 { for _, f := range applyMigrationFiles { pipe1 := pipep.New() go d.Migrate(f, pipe1) - if ok := pipep.WaitAndRedirect(pipe1, pipe, handleInterrupts()); !ok { + if ok := pipep.WaitAndRedirect(pipe1, pipe, signals); !ok { break } } From 91a33642328c0e6c42fc62b9c5026cc31f6ec4c8 Mon Sep 17 00:00:00 2001 From: Jeff Hodges Date: Fri, 8 Apr 2016 14:26:38 -0700 Subject: [PATCH 06/42] use a non-zero exit code when no command is given --- main.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index b6da438f..10925b10 100644 --- a/main.go +++ b/main.go @@ -158,7 +158,8 @@ func main() { fmt.Println(version) default: - fallthrough + helpCmd() + os.Exit(1) case "help": helpCmd() } From f230ced2017af333fb06c8b5c4dc0bd7adb8fc03 Mon Sep 17 00:00:00 2001 From: meehow Date: Sun, 15 May 2016 08:12:06 +0200 Subject: [PATCH 07/42] typo fixed --- driver/registry.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driver/registry.go b/driver/registry.go index 86a3955b..63889184 100644 --- a/driver/registry.go +++ b/driver/registry.go @@ -9,7 +9,7 @@ var driversMu sync.Mutex var drivers = make(map[string]Driver) // Registers a driver so it can be created from its name. Drivers should -// call this from an init() function so that they registers themselvse on +// call this from an init() function so that they registers themselves on // import func RegisterDriver(name string, driver Driver) { driversMu.Lock() From b5ef06870460f636c994d9af7590b4da34421980 Mon Sep 17 00:00:00 2001 From: Chris Gilling Date: Sun, 12 Jun 2016 14:59:23 -0700 Subject: [PATCH 08/42] cassandra: add error checking for bad connection urls --- docker-compose.yml | 2 +- driver/cassandra/cassandra.go | 7 +++++++ driver/cassandra/cassandra_test.go | 30 ++++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index cac3abcf..b1591c7a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,6 +21,6 @@ mysql: image: mysql environment: MYSQL_DATABASE: migratetest - MYSQL_ALLOW_EMPTY_PASSWORD: yes + MYSQL_ALLOW_EMPTY_PASSWORD: "yes" cassandra: image: cassandra:2.2 diff --git a/driver/cassandra/cassandra.go b/driver/cassandra/cassandra.go index e242de2d..f1891175 100644 --- a/driver/cassandra/cassandra.go +++ b/driver/cassandra/cassandra.go @@ -47,6 +47,13 @@ const ( // cassandra://localhost/SpaceOfKeys?protocol=4 func (driver *Driver) Initialize(rawurl string) error { u, err := url.Parse(rawurl) + if err != nil { + return fmt.Errorf("failed to parse connectil url: %v", err) + } + + if u.Path == "" { + return fmt.Errorf("no keyspace provided in connection url") + } cluster := gocql.NewCluster(u.Host) cluster.Keyspace = u.Path[1:len(u.Path)] diff --git a/driver/cassandra/cassandra_test.go b/driver/cassandra/cassandra_test.go index ea737945..2be8e2c1 100644 --- a/driver/cassandra/cassandra_test.go +++ b/driver/cassandra/cassandra_test.go @@ -112,5 +112,35 @@ func TestMigrate(t *testing.T) { if err := d.Close(); err != nil { t.Fatal(err) } +} + +func TestInitializeReturnsErrorsForBadUrls(t *testing.T) { + var session *gocql.Session + + host := os.Getenv("CASSANDRA_PORT_9042_TCP_ADDR") + port := os.Getenv("CASSANDRA_PORT_9042_TCP_PORT") + + cluster := gocql.NewCluster(host) + cluster.Consistency = gocql.All + cluster.Timeout = 1 * time.Minute + + session, err := cluster.CreateSession() + if err != nil { + t.Fatal(err) + } + defer session.Close() + if err := session.Query(`CREATE KEYSPACE IF NOT EXISTS migrate WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor': 1};`).Exec(); err != nil { + t.Fatal(err) + } + d := &Driver{} + invalidURL := "sdf://asdf://as?df?a" + if err := d.Initialize(invalidURL); err == nil { + t.Errorf("expected an error to be returned if url could not be parsed") + } + + noKeyspace := "cassandra://" + host + ":" + port + if err := d.Initialize(noKeyspace); err == nil { + t.Errorf("expected an error to be returned if no keyspace provided") + } } From 907d8cd935838b4f2a19ce309c84c4504d92bc09 Mon Sep 17 00:00:00 2001 From: Till Klocke Date: Thu, 16 Jun 2016 11:48:39 +0200 Subject: [PATCH 09/42] Added driver for crate.io database --- docker-compose.yml | 5 +- driver/crate/crate.go | 105 +++++++++++++++++++++++++++++++++++++ driver/crate/crate_test.go | 87 ++++++++++++++++++++++++++++++ main.go | 1 + 4 files changed, 197 insertions(+), 1 deletion(-) create mode 100644 driver/crate/crate.go create mode 100644 driver/crate/crate_test.go diff --git a/docker-compose.yml b/docker-compose.yml index cac3abcf..2b13d8f5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,6 +10,7 @@ go-test: - postgres - mysql - cassandra + - crate go-build: <<: *go command: sh -c 'go get -v && go build -ldflags ''-s'' -o migrater' @@ -21,6 +22,8 @@ mysql: image: mysql environment: MYSQL_DATABASE: migratetest - MYSQL_ALLOW_EMPTY_PASSWORD: yes + MYSQL_ALLOW_EMPTY_PASSWORD: "yes" cassandra: image: cassandra:2.2 +crate: + image: crate diff --git a/driver/crate/crate.go b/driver/crate/crate.go new file mode 100644 index 00000000..8773b665 --- /dev/null +++ b/driver/crate/crate.go @@ -0,0 +1,105 @@ +// Package crate implements a driver for the Crate.io database +package crate + +import ( + "database/sql" + "fmt" + "strings" + + _ "github.com/herenow/go-crate" + "github.com/mattes/migrate/driver" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" +) + +func init() { + driver.RegisterDriver("crate", &Driver{}) +} + +type Driver struct { + db *sql.DB +} + +const tableName = "schema_migrations" + +func (driver *Driver) Initialize(url string) error { + db, err := sql.Open("crate", url) + if err != nil { + return err + } + if err := db.Ping(); err != nil { + return err + } + driver.db = db + + if err := driver.ensureVersionTableExists(); err != nil { + return err + } + return nil +} + +func (driver *Driver) Close() error { + if err := driver.db.Close(); err != nil { + return err + } + return nil +} + +func (driver *Driver) FilenameExtension() string { + return "sql" +} + +func (driver *Driver) Version() (uint64, error) { + var version uint64 + err := driver.db.QueryRow("SELECT version FROM " + tableName + " ORDER BY version DESC LIMIT 1").Scan(&version) + switch { + case err == sql.ErrNoRows: + return 0, nil + case err != nil: + return 0, err + default: + return version, nil + } +} + +func (driver *Driver) Migrate(f file.File, pipe chan interface{}) { + defer close(pipe) + pipe <- f + + if err := f.ReadContent(); err != nil { + pipe <- err + return + } + + lines := strings.Split(string(f.Content), ";") + for _, line := range lines { + query := strings.TrimSpace(line) + query = strings.Replace(query, ";", "", -1) + if query != "" { + _, err := driver.db.Exec(query) + if err != nil { + pipe <- err + return + } + } + } + + if f.Direction == direction.Up { + if _, err := driver.db.Exec("INSERT INTO "+tableName+" (version) VALUES (?)", f.Version); err != nil { + pipe <- err + return + } + } else if f.Direction == direction.Down { + if _, err := driver.db.Exec("DELETE FROM "+tableName+" WHERE version=?", f.Version); err != nil { + pipe <- err + return + } + } +} + +func (driver *Driver) ensureVersionTableExists() error { + if _, err := driver.db.Exec(fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (version INTEGER PRIMARY KEY)", tableName)); err != nil { + return err + } + return nil +} diff --git a/driver/crate/crate_test.go b/driver/crate/crate_test.go new file mode 100644 index 00000000..05367ddc --- /dev/null +++ b/driver/crate/crate_test.go @@ -0,0 +1,87 @@ +package crate + +import ( + "fmt" + "os" + "testing" + + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" + pipep "github.com/mattes/migrate/pipe" +) + +func TestMigrate(t *testing.T) { + host := os.Getenv("CRATE_PORT_4200_TCP_ADDR") + port := os.Getenv("CRATE_PORT_4200_TCP_PORT") + + url := fmt.Sprintf("http://%s:%s", host, port) + + driver := &Driver{} + + if err := driver.Initialize(url); err != nil { + t.Fatal(err) + } + + successFiles := []file.File{ + { + Path: "/foobar", + FileName: "001_foobar.up.sql", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + CREATE TABLE yolo ( + id integer primary key, + msg string + ); + `), + }, + { + Path: "/foobar", + FileName: "002_foobar.down.sql", + Version: 1, + Name: "foobar", + Direction: direction.Down, + Content: []byte(` + DROP TABLE yolo; + `), + }, + } + + failFiles := []file.File{ + { + Path: "/foobar", + FileName: "002_foobar.up.sql", + Version: 2, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + CREATE TABLE error ( + id THIS WILL CAUSE AN ERROR + ) + `), + }, + } + + for _, file := range successFiles { + pipe := pipep.New() + go driver.Migrate(file, pipe) + errs := pipep.ReadErrors(pipe) + if len(errs) > 0 { + t.Fatal(errs) + } + } + + for _, file := range failFiles { + pipe := pipep.New() + go driver.Migrate(file, pipe) + errs := pipep.ReadErrors(pipe) + if len(errs) == 0 { + t.Fatal("Migration should have failed but succeeded") + } + } + + if err := driver.Close(); err != nil { + t.Fatal(err) + } +} diff --git a/main.go b/main.go index b6da438f..49f3e523 100644 --- a/main.go +++ b/main.go @@ -13,6 +13,7 @@ import ( "github.com/fatih/color" _ "github.com/mattes/migrate/driver/bash" _ "github.com/mattes/migrate/driver/cassandra" + _ "github.com/mattes/migrate/driver/crate" _ "github.com/mattes/migrate/driver/mysql" _ "github.com/mattes/migrate/driver/postgres" _ "github.com/mattes/migrate/driver/sqlite3" From a7fa9eb3fbac6dedba484ea47b770c2d1dae5d04 Mon Sep 17 00:00:00 2001 From: Till Klocke Date: Thu, 16 Jun 2016 13:08:04 +0200 Subject: [PATCH 10/42] Added README.md for Crate driver --- driver/crate/README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 driver/crate/README.md diff --git a/driver/crate/README.md b/driver/crate/README.md new file mode 100644 index 00000000..a43d0133 --- /dev/null +++ b/driver/crate/README.md @@ -0,0 +1,15 @@ +# Crate driver + +This is a driver for the [Crate](https://crate.io) database. It is based on the Crate +sql driver by [herenow](https://github.com/herenow/go-crate). + +This driver does not use transactions! This is not a limitation of the driver, but a +limitation of Crate. So handle situations with failed migrations with care! + +## Usage + +```bash +migrate -url http://host:port -path ./db/migrations create add_field_to_table +migrate -url http://host:port -path ./db/migrations up +migrate help # for more info +``` \ No newline at end of file From 6612806e067b1f502e4f40d489156e3bb7b53636 Mon Sep 17 00:00:00 2001 From: Till Klocke Date: Thu, 16 Jun 2016 13:47:21 +0200 Subject: [PATCH 11/42] Crate driver was using wrong url scheme. The migrate CLI never had a chance to select the correct driver --- driver/crate/crate.go | 1 + driver/crate/crate_test.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/driver/crate/crate.go b/driver/crate/crate.go index 8773b665..faf4ef77 100644 --- a/driver/crate/crate.go +++ b/driver/crate/crate.go @@ -23,6 +23,7 @@ type Driver struct { const tableName = "schema_migrations" func (driver *Driver) Initialize(url string) error { + url = strings.Replace(url, "crate", "http", 1) db, err := sql.Open("crate", url) if err != nil { return err diff --git a/driver/crate/crate_test.go b/driver/crate/crate_test.go index 05367ddc..18964328 100644 --- a/driver/crate/crate_test.go +++ b/driver/crate/crate_test.go @@ -14,7 +14,7 @@ func TestMigrate(t *testing.T) { host := os.Getenv("CRATE_PORT_4200_TCP_ADDR") port := os.Getenv("CRATE_PORT_4200_TCP_PORT") - url := fmt.Sprintf("http://%s:%s", host, port) + url := fmt.Sprintf("crate://%s:%s", host, port) driver := &Driver{} From cd5f75d47c7532396414617e83c533c5adf53d54 Mon Sep 17 00:00:00 2001 From: Till Klocke Date: Thu, 16 Jun 2016 14:34:50 +0200 Subject: [PATCH 12/42] Splitting of sql is now more robust and better tested --- driver/crate/crate.go | 27 ++++++++++++++++++--------- driver/crate/crate_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/driver/crate/crate.go b/driver/crate/crate.go index faf4ef77..eb30853a 100644 --- a/driver/crate/crate.go +++ b/driver/crate/crate.go @@ -72,16 +72,12 @@ func (driver *Driver) Migrate(f file.File, pipe chan interface{}) { return } - lines := strings.Split(string(f.Content), ";") + lines := splitContent(string(f.Content)) for _, line := range lines { - query := strings.TrimSpace(line) - query = strings.Replace(query, ";", "", -1) - if query != "" { - _, err := driver.db.Exec(query) - if err != nil { - pipe <- err - return - } + _, err := driver.db.Exec(line) + if err != nil { + pipe <- err + return } } @@ -98,6 +94,19 @@ func (driver *Driver) Migrate(f file.File, pipe chan interface{}) { } } +func splitContent(content string) []string { + lines := strings.Split(content, ";") + resultLines := make([]string, 0, len(lines)) + for i, line := range lines { + line = strings.Replace(lines[i], ";", "", -1) + line = strings.TrimSpace(line) + if line != "" { + resultLines = append(resultLines, line) + } + } + return resultLines +} + func (driver *Driver) ensureVersionTableExists() error { if _, err := driver.db.Exec(fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (version INTEGER PRIMARY KEY)", tableName)); err != nil { return err diff --git a/driver/crate/crate_test.go b/driver/crate/crate_test.go index 18964328..164cd57a 100644 --- a/driver/crate/crate_test.go +++ b/driver/crate/crate_test.go @@ -10,6 +10,30 @@ import ( pipep "github.com/mattes/migrate/pipe" ) +func TestContentSplit(t *testing.T) { + content := `CREATE TABLE users (user_id STRING primary key, first_name STRING, last_name STRING, email STRING, password_hash STRING) CLUSTERED INTO 3 shards WITH (number_of_replicas = 0); +CREATE TABLE units (unit_id STRING primary key, name STRING, members array(string)) CLUSTERED INTO 3 shards WITH (number_of_replicas = 0); +CREATE TABLE available_connectors (technology_id STRING primary key, description STRING, icon STRING, link STRING, configuration_parameters array(object as (name STRING, type STRING))) CLUSTERED INTO 3 shards WITH (number_of_replicas = 0); + ` + + lines := splitContent(content) + if len(lines) != 3 { + t.Errorf("Expected 3 lines, but got %d", len(lines)) + } + + if lines[0] != "CREATE TABLE users (user_id STRING primary key, first_name STRING, last_name STRING, email STRING, password_hash STRING) CLUSTERED INTO 3 shards WITH (number_of_replicas = 0)" { + t.Error("Line does not match expected output") + } + + if lines[1] != "CREATE TABLE units (unit_id STRING primary key, name STRING, members array(string)) CLUSTERED INTO 3 shards WITH (number_of_replicas = 0)" { + t.Error("Line does not match expected output") + } + + if lines[2] != "CREATE TABLE available_connectors (technology_id STRING primary key, description STRING, icon STRING, link STRING, configuration_parameters array(object as (name STRING, type STRING))) CLUSTERED INTO 3 shards WITH (number_of_replicas = 0)" { + t.Error("Line does not match expected output") + } +} + func TestMigrate(t *testing.T) { host := os.Getenv("CRATE_PORT_4200_TCP_ADDR") port := os.Getenv("CRATE_PORT_4200_TCP_PORT") From 42b45a6de8db3e4e0ce671fc895527b977903b42 Mon Sep 17 00:00:00 2001 From: Sandile Keswa Date: Sat, 23 Jul 2016 14:47:11 -0400 Subject: [PATCH 13/42] added consistency as an option --- driver/cassandra/cassandra.go | 41 +++++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/driver/cassandra/cassandra.go b/driver/cassandra/cassandra.go index e242de2d..aff35ae2 100644 --- a/driver/cassandra/cassandra.go +++ b/driver/cassandra/cassandra.go @@ -2,6 +2,7 @@ package cassandra import ( + "errors" "fmt" "net/url" "strconv" @@ -41,10 +42,12 @@ const ( ) // Cassandra Driver URL format: -// cassandra://host:port/keyspace?protocol=version +// cassandra://host:port/keyspace?protocol=version&consistency=level // -// Example: +// Examples: // cassandra://localhost/SpaceOfKeys?protocol=4 +// cassandra://localhost/SpaceOfKeys?protocol=4&consistency=all +// cassandra://localhost/SpaceOfKeys?consistency=quorum func (driver *Driver) Initialize(rawurl string) error { u, err := url.Parse(rawurl) @@ -53,6 +56,15 @@ func (driver *Driver) Initialize(rawurl string) error { cluster.Consistency = gocql.All cluster.Timeout = 1 * time.Minute + if len(u.Query().Get("consistency")) > 0 { + consistency, err := readConsistency(u.Query().Get("consistency")) + if err != nil { + return err + } + + cluster.Consistency = consistency + } + if len(u.Query().Get("protocol")) > 0 { protoversion, err := strconv.Atoi(u.Query().Get("protocol")) if err != nil { @@ -169,3 +181,28 @@ func (driver *Driver) Version() (uint64, error) { func init() { driver.RegisterDriver("cassandra", &Driver{}) } + +func readConsistency(consistency string) (gocql.Consistency, error) { + switch strings.ToLower(consistency) { + case "all": + return gocql.All, nil + case "each_quorum": + return gocql.EachQuorum, nil + case "quorum": + return gocql.Quorum, nil + case "local_quorum": + return gocql.LocalQuorum, nil + case "one": + return gocql.One, nil + case "two": + return gocql.Two, nil + case "three": + return gocql.Three, nil + case "local_one": + return gocql.LocalOne, nil + case "any": + return gocql.Any, nil + } + + return gocql.Consistency(-1), errors.New("Invalid consistency \"" + consistency + "\" specified") +} From cba0cc98474c8edc74179a00964f5e5a1c59ce8b Mon Sep 17 00:00:00 2001 From: Sandile Keswa Date: Sat, 23 Jul 2016 15:04:11 -0400 Subject: [PATCH 14/42] fixed the overflow --- driver/cassandra/cassandra.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driver/cassandra/cassandra.go b/driver/cassandra/cassandra.go index aff35ae2..9850639f 100644 --- a/driver/cassandra/cassandra.go +++ b/driver/cassandra/cassandra.go @@ -204,5 +204,5 @@ func readConsistency(consistency string) (gocql.Consistency, error) { return gocql.Any, nil } - return gocql.Consistency(-1), errors.New("Invalid consistency \"" + consistency + "\" specified") + return gocql.Consistency(0), errors.New("Invalid consistency \"" + consistency + "\" specified") } From 0cd470ee98ec0d4905457058077df0ca51b0993a Mon Sep 17 00:00:00 2001 From: Sandile Keswa Date: Mon, 25 Jul 2016 12:02:50 -0400 Subject: [PATCH 15/42] Adopted @nlacey's suggestion. --- driver/cassandra/cassandra.go | 39 ++++++++++++++--------------------- 1 file changed, 15 insertions(+), 24 deletions(-) diff --git a/driver/cassandra/cassandra.go b/driver/cassandra/cassandra.go index 9850639f..7bcef74c 100644 --- a/driver/cassandra/cassandra.go +++ b/driver/cassandra/cassandra.go @@ -2,7 +2,6 @@ package cassandra import ( - "errors" "fmt" "net/url" "strconv" @@ -57,7 +56,7 @@ func (driver *Driver) Initialize(rawurl string) error { cluster.Timeout = 1 * time.Minute if len(u.Query().Get("consistency")) > 0 { - consistency, err := readConsistency(u.Query().Get("consistency")) + consistency, err := parseConsistency(u.Query().Get("consistency")) if err != nil { return err } @@ -182,27 +181,19 @@ func init() { driver.RegisterDriver("cassandra", &Driver{}) } -func readConsistency(consistency string) (gocql.Consistency, error) { - switch strings.ToLower(consistency) { - case "all": - return gocql.All, nil - case "each_quorum": - return gocql.EachQuorum, nil - case "quorum": - return gocql.Quorum, nil - case "local_quorum": - return gocql.LocalQuorum, nil - case "one": - return gocql.One, nil - case "two": - return gocql.Two, nil - case "three": - return gocql.Three, nil - case "local_one": - return gocql.LocalOne, nil - case "any": - return gocql.Any, nil - } +// ParseConsistency wraps gocql.ParseConsistency to return an error +// instead of a panicing. +func parseConsistency(consistencyStr string) (consistency gocql.Consistency, err error) { + defer func() { + if r := recover(); r != nil { + var ok bool + err, ok = r.(error) + if !ok { + err = fmt.Errorf("Failed to parse consistency \"%s\": %v", s, r) + } + } + }() + consistency = gocql.ParseConsistency(consistencyStr) - return gocql.Consistency(0), errors.New("Invalid consistency \"" + consistency + "\" specified") + return consistency, nil } From aa4dc1252b9607c67f6facdf6ee32e5c9d265c0d Mon Sep 17 00:00:00 2001 From: Sandile Keswa Date: Mon, 25 Jul 2016 13:25:45 -0400 Subject: [PATCH 16/42] Oops! Correcting a renaming error. --- driver/cassandra/cassandra.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driver/cassandra/cassandra.go b/driver/cassandra/cassandra.go index 7bcef74c..7fb77412 100644 --- a/driver/cassandra/cassandra.go +++ b/driver/cassandra/cassandra.go @@ -189,7 +189,7 @@ func parseConsistency(consistencyStr string) (consistency gocql.Consistency, err var ok bool err, ok = r.(error) if !ok { - err = fmt.Errorf("Failed to parse consistency \"%s\": %v", s, r) + err = fmt.Errorf("Failed to parse consistency \"%s\": %v", consistencyStr, r) } } }() From 8630ea78b78799bd69b6081f18b6ba2652c66709 Mon Sep 17 00:00:00 2001 From: dimag Date: Thu, 4 Aug 2016 15:52:31 +0300 Subject: [PATCH 17/42] changed import paths to point to dimag-jfrog --- driver/bash/bash.go | 4 ++-- driver/cassandra/cassandra.go | 6 +++--- driver/cassandra/cassandra_test.go | 6 +++--- driver/driver.go | 2 +- driver/mysql/mysql.go | 6 +++--- driver/mysql/mysql_test.go | 6 +++--- driver/postgres/postgres.go | 6 +++--- driver/postgres/postgres_test.go | 6 +++--- driver/sqlite3/sqlite3.go | 6 +++--- driver/sqlite3/sqlite3_test.go | 6 +++--- file/file.go | 2 +- file/file_test.go | 2 +- main.go | 20 ++++++++++---------- migrate/migrate.go | 8 ++++---- migrate/migrate_test.go | 4 ++-- 15 files changed, 45 insertions(+), 45 deletions(-) diff --git a/driver/bash/bash.go b/driver/bash/bash.go index 031f9bb4..97ba91f1 100644 --- a/driver/bash/bash.go +++ b/driver/bash/bash.go @@ -2,8 +2,8 @@ package bash import ( - "github.com/mattes/migrate/driver" - "github.com/mattes/migrate/file" + "github.com/dimag-jfrog/migrate/driver" + "github.com/dimag-jfrog/migrate/file" ) type Driver struct { diff --git a/driver/cassandra/cassandra.go b/driver/cassandra/cassandra.go index e242de2d..ffc76ada 100644 --- a/driver/cassandra/cassandra.go +++ b/driver/cassandra/cassandra.go @@ -9,9 +9,9 @@ import ( "time" "github.com/gocql/gocql" - "github.com/mattes/migrate/driver" - "github.com/mattes/migrate/file" - "github.com/mattes/migrate/migrate/direction" + "github.com/dimag-jfrog/migrate/driver" + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/migrate/direction" ) type Driver struct { diff --git a/driver/cassandra/cassandra_test.go b/driver/cassandra/cassandra_test.go index ea737945..7d363895 100644 --- a/driver/cassandra/cassandra_test.go +++ b/driver/cassandra/cassandra_test.go @@ -7,9 +7,9 @@ import ( "time" "github.com/gocql/gocql" - "github.com/mattes/migrate/file" - "github.com/mattes/migrate/migrate/direction" - pipep "github.com/mattes/migrate/pipe" + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/migrate/direction" + pipep "github.com/dimag-jfrog/migrate/pipe" ) func TestMigrate(t *testing.T) { diff --git a/driver/driver.go b/driver/driver.go index e4ecb783..5c2f6fcc 100644 --- a/driver/driver.go +++ b/driver/driver.go @@ -5,7 +5,7 @@ import ( "fmt" neturl "net/url" // alias to allow `url string` func signature in New - "github.com/mattes/migrate/file" + "github.com/dimag-jfrog/migrate/file" ) // Driver is the interface type that needs to implemented by all drivers. diff --git a/driver/mysql/mysql.go b/driver/mysql/mysql.go index 2bbbc131..cf186f8f 100644 --- a/driver/mysql/mysql.go +++ b/driver/mysql/mysql.go @@ -12,9 +12,9 @@ import ( "strings" "github.com/go-sql-driver/mysql" - "github.com/mattes/migrate/driver" - "github.com/mattes/migrate/file" - "github.com/mattes/migrate/migrate/direction" + "github.com/dimag-jfrog/migrate/driver" + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/migrate/direction" ) type Driver struct { diff --git a/driver/mysql/mysql_test.go b/driver/mysql/mysql_test.go index 51cf552a..9e08af9f 100644 --- a/driver/mysql/mysql_test.go +++ b/driver/mysql/mysql_test.go @@ -6,9 +6,9 @@ import ( "strings" "testing" - "github.com/mattes/migrate/file" - "github.com/mattes/migrate/migrate/direction" - pipep "github.com/mattes/migrate/pipe" + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/migrate/direction" + pipep "github.com/dimag-jfrog/migrate/pipe" ) // TestMigrate runs some additional tests on Migrate(). diff --git a/driver/postgres/postgres.go b/driver/postgres/postgres.go index 502c7d1a..dbbe4a26 100644 --- a/driver/postgres/postgres.go +++ b/driver/postgres/postgres.go @@ -8,9 +8,9 @@ import ( "strconv" "github.com/lib/pq" - "github.com/mattes/migrate/driver" - "github.com/mattes/migrate/file" - "github.com/mattes/migrate/migrate/direction" + "github.com/dimag-jfrog/migrate/driver" + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/migrate/direction" ) type Driver struct { diff --git a/driver/postgres/postgres_test.go b/driver/postgres/postgres_test.go index ec91b350..552f23d3 100644 --- a/driver/postgres/postgres_test.go +++ b/driver/postgres/postgres_test.go @@ -5,9 +5,9 @@ import ( "os" "testing" - "github.com/mattes/migrate/file" - "github.com/mattes/migrate/migrate/direction" - pipep "github.com/mattes/migrate/pipe" + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/migrate/direction" + pipep "github.com/dimag-jfrog/migrate/pipe" ) // TestMigrate runs some additional tests on Migrate(). diff --git a/driver/sqlite3/sqlite3.go b/driver/sqlite3/sqlite3.go index a92b14d8..65e6a2ac 100644 --- a/driver/sqlite3/sqlite3.go +++ b/driver/sqlite3/sqlite3.go @@ -7,9 +7,9 @@ import ( "fmt" "strings" - "github.com/mattes/migrate/driver" - "github.com/mattes/migrate/file" - "github.com/mattes/migrate/migrate/direction" + "github.com/dimag-jfrog/migrate/driver" + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/migrate/direction" "github.com/mattn/go-sqlite3" ) diff --git a/driver/sqlite3/sqlite3_test.go b/driver/sqlite3/sqlite3_test.go index 8d51a7de..29a501f2 100644 --- a/driver/sqlite3/sqlite3_test.go +++ b/driver/sqlite3/sqlite3_test.go @@ -4,9 +4,9 @@ import ( "database/sql" "testing" - "github.com/mattes/migrate/file" - "github.com/mattes/migrate/migrate/direction" - pipep "github.com/mattes/migrate/pipe" + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/migrate/direction" + pipep "github.com/dimag-jfrog/migrate/pipe" ) // TestMigrate runs some additional tests on Migrate() diff --git a/file/file.go b/file/file.go index 54992a48..14617ba6 100644 --- a/file/file.go +++ b/file/file.go @@ -5,7 +5,7 @@ import ( "bytes" "errors" "fmt" - "github.com/mattes/migrate/migrate/direction" + "github.com/dimag-jfrog/migrate/migrate/direction" "go/token" "io/ioutil" "path" diff --git a/file/file_test.go b/file/file_test.go index b9ddeabd..c6bd0b3c 100644 --- a/file/file_test.go +++ b/file/file_test.go @@ -1,7 +1,7 @@ package file import ( - "github.com/mattes/migrate/migrate/direction" + "github.com/dimag-jfrog/migrate/migrate/direction" "io/ioutil" "os" "path" diff --git a/main.go b/main.go index b6da438f..51d68a84 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,6 @@ // Package main is the CLI. // You can use the CLI via Terminal. -// import "github.com/mattes/migrate/migrate" for usage within Go. +// import "github.com/dimag-jfrog/migrate/migrate" for usage within Go. package main import ( @@ -11,15 +11,15 @@ import ( "time" "github.com/fatih/color" - _ "github.com/mattes/migrate/driver/bash" - _ "github.com/mattes/migrate/driver/cassandra" - _ "github.com/mattes/migrate/driver/mysql" - _ "github.com/mattes/migrate/driver/postgres" - _ "github.com/mattes/migrate/driver/sqlite3" - "github.com/mattes/migrate/file" - "github.com/mattes/migrate/migrate" - "github.com/mattes/migrate/migrate/direction" - pipep "github.com/mattes/migrate/pipe" + _ "github.com/dimag-jfrog/migrate/driver/bash" + _ "github.com/dimag-jfrog/migrate/driver/cassandra" + _ "github.com/dimag-jfrog/migrate/driver/mysql" + _ "github.com/dimag-jfrog/migrate/driver/postgres" + _ "github.com/dimag-jfrog/migrate/driver/sqlite3" + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/migrate" + "github.com/dimag-jfrog/migrate/migrate/direction" + pipep "github.com/dimag-jfrog/migrate/pipe" ) var url = flag.String("url", os.Getenv("MIGRATE_URL"), "") diff --git a/migrate/migrate.go b/migrate/migrate.go index b452aeef..54cc42bc 100644 --- a/migrate/migrate.go +++ b/migrate/migrate.go @@ -11,10 +11,10 @@ import ( "strconv" "strings" - "github.com/mattes/migrate/driver" - "github.com/mattes/migrate/file" - "github.com/mattes/migrate/migrate/direction" - pipep "github.com/mattes/migrate/pipe" + "github.com/dimag-jfrog/migrate/driver" + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/migrate/direction" + pipep "github.com/dimag-jfrog/migrate/pipe" ) // Up applies all available migrations diff --git a/migrate/migrate_test.go b/migrate/migrate_test.go index 7d2a622f..7909826d 100644 --- a/migrate/migrate_test.go +++ b/migrate/migrate_test.go @@ -6,8 +6,8 @@ import ( "testing" // Ensure imports for each driver we wish to test - _ "github.com/mattes/migrate/driver/postgres" - _ "github.com/mattes/migrate/driver/sqlite3" + _ "github.com/dimag-jfrog/migrate/driver/postgres" + _ "github.com/dimag-jfrog/migrate/driver/sqlite3" ) // Add Driver URLs here to test basic Up, Down, .. functions. From 64101a13e946a05197e0ff6b5c589827544f2ffc Mon Sep 17 00:00:00 2001 From: dimag Date: Mon, 8 Aug 2016 16:29:25 +0300 Subject: [PATCH 18/42] Changes that interfere with the Migrate open source code: - Added go methods migrator, mongo db template: different from the usual driver model. - Added support for bidirectional files (for go methods), appending _up or _down upon context - Added DriverWithFilnameParser for providing custom filename parser functionality that knows to parse bi-directional file names. --- .gitignore | 11 +- driver/driver.go | 9 + driver/gomethods/gomethods_migrator.go | 176 +++++++++ driver/gomethods/gomethods_migrator_test.go | 342 ++++++++++++++++++ driver/gomethods/mongodb/mongodb_template.go | 111 ++++++ driver/gomethods/usage_examples/mongodb.go | 121 +++++++ .../gomethods/usage_examples/mongodb_test.go | 123 +++++++ file/file.go | 106 +++--- file/file_test.go | 50 +-- file/filename_parser.go | 85 +++++ file/filename_parser_test.go | 85 +++++ migrate/direction/direction.go | 1 + migrate/migrate.go | 12 +- 13 files changed, 1128 insertions(+), 104 deletions(-) create mode 100644 driver/gomethods/gomethods_migrator.go create mode 100644 driver/gomethods/gomethods_migrator_test.go create mode 100644 driver/gomethods/mongodb/mongodb_template.go create mode 100644 driver/gomethods/usage_examples/mongodb.go create mode 100644 driver/gomethods/usage_examples/mongodb_test.go create mode 100644 file/filename_parser.go create mode 100644 file/filename_parser_test.go diff --git a/.gitignore b/.gitignore index ee2f41ff..9bd9f472 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,11 @@ .DS_Store -test.db \ No newline at end of file +test.db +*.iml +.DS_Store +.idea +*.zip +*.tar.* +*.nuget +*.jar +*.war + diff --git a/driver/driver.go b/driver/driver.go index 5c2f6fcc..a668d52e 100644 --- a/driver/driver.go +++ b/driver/driver.go @@ -34,6 +34,15 @@ type Driver interface { Version() (uint64, error) } +// Driver that has some custom migration file format +// and wants to use a different parsing strategy. +type DriverWithFilenameParser interface { + + Driver + + FilenameParser() file.FilenameParser +} + // New returns Driver and calls Initialize on it func New(url string) (Driver, error) { u, err := neturl.Parse(url) diff --git a/driver/gomethods/gomethods_migrator.go b/driver/gomethods/gomethods_migrator.go new file mode 100644 index 00000000..68b9ff52 --- /dev/null +++ b/driver/gomethods/gomethods_migrator.go @@ -0,0 +1,176 @@ +package gomethods + +import ( + //"bytes" + "reflect" + "fmt" + "strings" + "os" + "path" + "bufio" + "github.com/dimag-jfrog/migrate/driver" + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/migrate/direction" +) + + +type MissingMethodError string +func (e MissingMethodError) Error() string { return "Non existing migrate method: " + string(e) } + + +type WrongMethodSignatureError string +func (e WrongMethodSignatureError) Error() string { return fmt.Sprintf("Method %s has wrong signature", e) } + +type MethodInvocationFailedError struct { + MethodName string + Err error +} + +func (e *MethodInvocationFailedError) Error() string { + return fmt.Sprintf("Method %s returned an error: %v", e.MethodName, e.Error) +} + + +type Migrator struct { + Driver driver.DriverWithFilenameParser + RollbackOnFailure bool +} + +func (m *Migrator) Migrate(f file.File, pipe chan interface{}) error { + methods, err := m.getMigrationMethods(f) + if err != nil { + pipe <- err + return err + } + + for i, methodName := range methods { + pipe <- methodName + err := m.Invoke(methodName) + if err != nil { + pipe <- err + if !m.RollbackOnFailure { + return err + } + + // on failure, try to rollback methods in this migration + for j := i-1; j >= 0; j-- { + rollbackToMethodName := getRollbackToMethod(methods[j]) + pipe <- rollbackToMethodName + err = m.Invoke(rollbackToMethodName) + if err != nil { + pipe <- err + break + } + } + return err + } + } + + return nil +} + +func (m *Migrator) IsValid(methodName string) bool { + return reflect.ValueOf(m.Driver).MethodByName(methodName).IsValid() +} + +func (m *Migrator) Invoke(methodName string) error { + name := methodName + migrateMethod := reflect.ValueOf(m.Driver).MethodByName(name) + if !migrateMethod.IsValid() { + return MissingMethodError(methodName) + } + + retValues := migrateMethod.Call([]reflect.Value{}) + if len(retValues) != 1 { + return WrongMethodSignatureError(name) + } + + if !retValues[0].IsNil() { + err, ok := retValues[0].Interface().(error) + if !ok { + return WrongMethodSignatureError(name) + } + return &MethodInvocationFailedError{ MethodName:name, Err:err} + } + + return nil +} + +func reverseInPlace(a []string) { + for i := 0; i < len(a)/2; i++ { + j := len(a) - i - 1 + a[i], a[j] = a[j], a[i] + } +} + +func getRollbackToMethod(methodName string) string { + if strings.HasSuffix(methodName, "_up") { + return strings.TrimSuffix(methodName, "_up") + "_down" + } else { + return strings.TrimSuffix(methodName, "_down") + "_up" + } +} + +func getFileLines(file file.File) ([]string, error) { + if len(file.Content) == 0 { + lines := make([]string, 0) + file, err := os.Open(path.Join(file.Path, file.FileName)) + if err != nil { + return nil, err + } + defer file.Close() + + scanner := bufio.NewScanner(file) + + for scanner.Scan() { + lines = append(lines, scanner.Text()) + } + return lines, nil + } else { + //n := bytes.IndexByte(file.Content, 0) + //n := bytes.Index(file.Content, []byte{0}) + //s := string(file.Content[:n]) + s := string(file.Content) + return strings.Split(s, "\n"), nil + } +} + +func (m *Migrator) getMigrationMethods(f file.File) ([]string, error) { + var lines, methods []string + lines, err := getFileLines(f) + if err != nil { + return nil, err + } + + for _, line := range lines { + operationName := strings.TrimSpace(line) + + if operationName == "" || strings.HasPrefix(operationName, "--") { + // an empty line or a comment, ignore + continue + } + + upMethodName := operationName + "_up" + downMethodName := operationName + "_down" + + if !m.IsValid(upMethodName) { + return nil, MissingMethodError(upMethodName) + } + if !m.IsValid(downMethodName) { + return nil, MissingMethodError(downMethodName) + } + + if f.Direction == direction.Up { + methods = append(methods, upMethodName) + } else { + methods = append(methods, downMethodName) + } + } + + _,_,fileType,_ := m.Driver.FilenameParser().Parse(f.FileName) + if fileType == direction.Both && f.Direction == direction.Down { + reverseInPlace(methods) + } + return methods, nil + +} diff --git a/driver/gomethods/gomethods_migrator_test.go b/driver/gomethods/gomethods_migrator_test.go new file mode 100644 index 00000000..58fd4f2b --- /dev/null +++ b/driver/gomethods/gomethods_migrator_test.go @@ -0,0 +1,342 @@ +package gomethods + +import ( + "reflect" + "testing" + + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/migrate/direction" + + pipep "github.com/dimag-jfrog/migrate/pipe" +) + +type FakeGoMethodsDriver struct { + InvokedMethods []string + Migrator Migrator +} + +func (driver *FakeGoMethodsDriver) Initialize(url string) error { + return nil +} + +func (driver *FakeGoMethodsDriver) Close() error { + return nil +} + +func (driver *FakeGoMethodsDriver) FilenameParser() file.FilenameParser { + return file.UpDownAndBothFilenameParser{ FilenameExtension: driver.FilenameExtension() } +} + +func (driver *FakeGoMethodsDriver) FilenameExtension() string { + return "gm" +} + +func (driver *FakeGoMethodsDriver) Version() (uint64, error) { + return uint64(0), nil +} + +func (driver *FakeGoMethodsDriver) Migrate(f file.File, pipe chan interface{}) { + defer close(pipe) + pipe <- f + return +} + +func (driver *FakeGoMethodsDriver) V001_init_organizations_up() error { + driver.InvokedMethods = append(driver.InvokedMethods, "V001_init_organizations_up") + return nil +} + +func (driver *FakeGoMethodsDriver) V001_init_organizations_down() error { + driver.InvokedMethods = append(driver.InvokedMethods, "V001_init_organizations_down") + return nil + +} + +func (driver *FakeGoMethodsDriver) V001_init_users_up() error { + driver.InvokedMethods = append(driver.InvokedMethods, "V001_init_users_up") + return nil +} + +func (driver *FakeGoMethodsDriver) V001_init_users_down() error { + driver.InvokedMethods = append(driver.InvokedMethods, "V001_init_users_down") + return nil +} + +type SomeError struct{} +func (e SomeError) Error() string { return "Some error happened" } + +func (driver *FakeGoMethodsDriver) V001_some_failing_method_up() error { + driver.InvokedMethods = append(driver.InvokedMethods, "V001_some_failing_method_up") + return SomeError{} +} + +func (driver *FakeGoMethodsDriver) V001_some_failing_method_down() error { + driver.InvokedMethods = append(driver.InvokedMethods, "V001_some_failing_method_down") + return SomeError{} +} + +func TestMigrate(t *testing.T) { + cases := []struct { + name string + file file.File + expectedInvokedMethods []string + expectedErrors []error + expectRollback bool + }{ + { + name: "up migration, both directions-file: invokes up methods in order", + file: file.File { + Path: "/foobar", + FileName: "001_foobar.gm", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V001_init_organizations + V001_init_users + `), + }, + expectedInvokedMethods: []string{"V001_init_organizations_up", "V001_init_users_up"}, + expectedErrors: []error{}, + }, + { + name: "down migration, both-directions-file: reverts direction of invoked down methods", + file: file.File { + Path: "/foobar", + FileName: "001_foobar.gm", + Version: 1, + Name: "foobar", + Direction: direction.Down, + Content: []byte(` + V001_init_organizations + V001_init_users + `), + }, + expectedInvokedMethods: []string{"V001_init_users_down", "V001_init_organizations_down"}, + expectedErrors: []error{}, + }, + { + name: "up migration, up direction-file: invokes up methods in order", + file: file.File { + Path: "/foobar", + FileName: "001_foobar.up.gm", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V001_init_organizations + V001_init_users + `), + }, + expectedInvokedMethods: []string{"V001_init_organizations_up", "V001_init_users_up"}, + expectedErrors: []error{}, + }, + { + name: "down migration, down directions-file: keeps order of invoked down methods", + file: file.File { + Path: "/foobar", + FileName: "001_foobar.down.gm", + Version: 1, + Name: "foobar", + Direction: direction.Down, + Content: []byte(` + V001_init_organizations + V001_init_users + `), + }, + expectedInvokedMethods: []string{"V001_init_organizations_down", "V001_init_users_down"}, + expectedErrors: []error{}, + }, + { + name: "up migration: non-existing method causes migration not to execute", + file: file.File { + Path: "/foobar", + FileName: "001_foobar.gm", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V001_init_organizations + V001_init_users + V001_some_non_existing_method + `), + }, + expectedInvokedMethods: []string{}, + expectedErrors: []error{ MissingMethodError("V001_some_non_existing_method_up") }, + }, + { + name: "up migration: failing method stops execution", + file: file.File { + Path: "/foobar", + FileName: "001_foobar.gm", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V001_init_organizations + V001_some_failing_method + V001_init_users + `), + }, + expectedInvokedMethods: []string{ + "V001_init_organizations_up", + "V001_some_failing_method_up", + }, + expectedErrors: []error{ &MethodInvocationFailedError{ + MethodName: "V001_some_failing_method_up", + Err: SomeError{}, + }}, + }, + { + name: "down migration, both-directions-file: failing method stops migration", + file: file.File { + Path: "/foobar", + FileName: "001_foobar.gm", + Version: 1, + Name: "foobar", + Direction: direction.Down, + Content: []byte(` + V001_init_organizations + V001_some_failing_method + V001_init_users + `), + }, + expectedInvokedMethods: []string{ + "V001_init_users_down", + "V001_some_failing_method_down", + }, + expectedErrors: []error{ &MethodInvocationFailedError{ + MethodName: "V001_some_failing_method_down", + Err: SomeError{}, + }}, + }, + { + name: "up migration: failing method causes rollback in rollback mode", + expectRollback: true, + file: file.File { + Path: "/foobar", + FileName: "001_foobar.gm", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V001_init_organizations + V001_init_users + V001_some_failing_method + `), + }, + expectedInvokedMethods: []string{ + "V001_init_organizations_up", + "V001_init_users_up", + "V001_some_failing_method_up", + "V001_init_users_down", + "V001_init_organizations_down", + }, + expectedErrors: []error{ &MethodInvocationFailedError{ + MethodName: "V001_some_failing_method_up", + Err: SomeError{}, + }}, + }, + { + name: "down migration, both-directions-file: failing method causes rollback in rollback mode", + expectRollback: true, + file: file.File { + Path: "/foobar", + FileName: "001_foobar.gm", + Version: 1, + Name: "foobar", + Direction: direction.Down, + Content: []byte(` + V001_init_organizations + V001_some_failing_method + V001_init_users + `), + }, + expectedInvokedMethods: []string{ + "V001_init_users_down", + "V001_some_failing_method_down", + "V001_init_users_up", + }, + expectedErrors: []error{ &MethodInvocationFailedError{ + MethodName: "V001_some_failing_method_down", + Err: SomeError{}, + }}, + }, + + } + + for _, c := range cases { + migrator := Migrator{} + d := &FakeGoMethodsDriver{Migrator: migrator, InvokedMethods:[]string{}} + migrator.Driver = d + migrator.RollbackOnFailure = c.expectRollback + + pipe := pipep.New() + go func() { + migrator.Migrate(c.file, pipe) + close(pipe) + }() + errs := pipep.ReadErrors(pipe) + + var failed bool + if !reflect.DeepEqual(d.InvokedMethods, c.expectedInvokedMethods) { + failed = true + t.Errorf("case '%s': FAILED\nexpected invoked methods %v\nbut got %v", c.name, c.expectedInvokedMethods, d.InvokedMethods) + } + if !reflect.DeepEqual(errs, c.expectedErrors) { + failed = true + t.Errorf("case '%s': FAILED\nexpected errors %v\nbut got %v", c.name, c.expectedErrors, errs) + + } + if !failed { + t.Logf("case '%s': PASSED", c.name) + } + } +} + + + +func TestGetRollbackToMethod(t *testing.T) { + cases := []struct { + method string + expectedRollbackMethod string + }{ + {"some_method_up", "some_method_down"}, + {"some_method_down", "some_method_up"}, + {"up_down_up", "up_down_down"}, + {"down_up", "down_down"}, + {"down_down", "down_up"}, + } + + for _, c := range cases { + actualRollbackMethod := getRollbackToMethod(c.method) + if actualRollbackMethod != c.expectedRollbackMethod { + t.Errorf("Expected rollback method to be %s but got %s", c.expectedRollbackMethod, actualRollbackMethod) + } + } +} + +func TestReverseInPlace(t *testing.T) { + methods := []string { + "method1_down", + "method2_down", + "method3_down", + "method4_down", + "method5_down", + } + + expectedReversedMethods := []string { + "method5_down", + "method4_down", + "method3_down", + "method2_down", + "method1_down", + } + + reverseInPlace(methods) + + if !reflect.DeepEqual(methods, expectedReversedMethods) { + t.Errorf("Expected reverse methods %v but got %v", expectedReversedMethods, methods) + } +} + diff --git a/driver/gomethods/mongodb/mongodb_template.go b/driver/gomethods/mongodb/mongodb_template.go new file mode 100644 index 00000000..48817cf6 --- /dev/null +++ b/driver/gomethods/mongodb/mongodb_template.go @@ -0,0 +1,111 @@ +package mongodb + +import ( + "gopkg.in/mgo.v2" + "gopkg.in/mgo.v2/bson" + "errors" + "strings" + "github.com/dimag-jfrog/migrate/migrate/direction" + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/driver/gomethods" +) + +const MIGRATE_C = "db_migrations" + + +// This is not a real driver since the Initialize method requires a gomethods.Migrator +// The real driver will contain the DriverTemplate and implement all the custom migration Golang methods +// See example in usage_examples for details +type DriverTemplate struct { + Session *mgo.Session + DbName string + + migrator gomethods.Migrator +} + + +type DbMigration struct { + Id bson.ObjectId `bson:"_id,omitempty"` + Version uint64 `bson:"version"` +} + +func (driver *DriverTemplate) Initialize(url, dbName string, migrator gomethods.Migrator) error { + urlWithoutScheme := strings.SplitN(url, "mongodb://", 2) + if len(urlWithoutScheme) != 2 { + return errors.New("invalid mongodb:// scheme") + } + + session, err := mgo.Dial(url) + if err != nil { + return err + } + session.SetMode(mgo.Monotonic, true) + + driver.Session = session + driver.DbName = dbName + driver.migrator = migrator + + return nil +} + +func (driver *DriverTemplate) Close() error { + if driver.Session != nil { + driver.Session.Close() + } + return nil +} + +func (driver *DriverTemplate) FilenameParser() file.FilenameParser { + return file.UpDownAndBothFilenameParser{FilenameExtension: driver.FilenameExtension()} +} + +func (driver *DriverTemplate) FilenameExtension() string { + return "mgo" +} + + +func (driver *DriverTemplate) Version() (uint64, error) { + var latestMigration DbMigration + c := driver.Session.DB(driver.DbName).C(MIGRATE_C) + + + err := c.Find(bson.M{}).Sort("-version").One(&latestMigration) + + switch { + case err == mgo.ErrNotFound: + return 0, nil + case err != nil: + return 0, err + default: + return latestMigration.Version, nil + } +} +func (driver *DriverTemplate) Migrate(f file.File, pipe chan interface{}) { + defer close(pipe) + pipe <- f + + err := driver.migrator.Migrate(f, pipe) + if err != nil { + return + } + + migrate_c := driver.Session.DB(driver.DbName).C(MIGRATE_C) + + if f.Direction == direction.Up { + id := bson.NewObjectId() + dbMigration := DbMigration{Id: id, Version: f.Version} + + err := migrate_c.Insert(dbMigration) + if err != nil { + pipe <- err + return + } + + } else if f.Direction == direction.Down { + err := migrate_c.Remove(bson.M{"version": f.Version}) + if err != nil { + pipe <- err + return + } + } +} diff --git a/driver/gomethods/usage_examples/mongodb.go b/driver/gomethods/usage_examples/mongodb.go new file mode 100644 index 00000000..be0ccb2d --- /dev/null +++ b/driver/gomethods/usage_examples/mongodb.go @@ -0,0 +1,121 @@ +package usage_examples + +import ( + "github.com/dimag-jfrog/migrate/driver" + "github.com/dimag-jfrog/migrate/driver/gomethods" + "github.com/dimag-jfrog/migrate/driver/gomethods/mongodb" + "gopkg.in/mgo.v2/bson" + "time" +) + +// This boilerplate part is necessary and the same +// regardless of the specific mongodb golang methods driver + +type GoMethodsMongoDbDriver struct { + mongodb.DriverTemplate +} + +func (d *GoMethodsMongoDbDriver) Initialize(url string) error { + return d.DriverTemplate.Initialize(url, DB_NAME, gomethods.Migrator{Driver: d}) +} + +func init() { + driver.RegisterDriver("mongodb", &GoMethodsMongoDbDriver{mongodb.DriverTemplate{}}) +} + + + +// Here goes the specific mongodb golang methods driver logic + +const DB_NAME = "test" +const SHORT_DATE_LAYOUT = "2000-Jan-01" +const USERS_C = "users" +const ORGANIZATIONS_C = "organizations" + +type Organization struct { + Id bson.ObjectId `bson:"_id,omitempty"` + Name string `bson:"name"` + Location string `bson:"location"` + DateFounded time.Time `bson:"date_founded"` +} + +type User struct { + Id bson.ObjectId `bson:"_id"` + Name string `bson:"name"` +} + +func (m *GoMethodsMongoDbDriver) V001_init_organizations_up() error { + date1, _ := time.Parse(SHORT_DATE_LAYOUT, "1994-Jul-05") + date2, _ := time.Parse(SHORT_DATE_LAYOUT, "1998-Sep-04") + date3, _ := time.Parse(SHORT_DATE_LAYOUT, "2008-Apr-28") + + orgs := []Organization{ + {Id: bson.NewObjectId(), Name: "Amazon", Location:"Seattle", DateFounded: date1}, + {Id: bson.NewObjectId(), Name: "Google", Location:"Mountain View", DateFounded: date2}, + {Id: bson.NewObjectId(), Name: "JFrog", Location:"Santa Clara", DateFounded: date3}, + } + + for _, org := range orgs { + err := m.Session.DB(DB_NAME).C(ORGANIZATIONS_C).Insert(org) + if err != nil { + return err + } + } + return nil +} + +func (m *GoMethodsMongoDbDriver) V001_init_organizations_down() error { + return m.Session.DB(DB_NAME).C(ORGANIZATIONS_C).DropCollection() +} + +func (m *GoMethodsMongoDbDriver) V001_init_users_up() error { + users := []User{ + {Id: bson.NewObjectId(), Name: "Alex"}, + {Id: bson.NewObjectId(), Name: "Beatrice"}, + {Id: bson.NewObjectId(), Name: "Cleo"}, + } + + for _, user := range users { + err := m.Session.DB(DB_NAME).C(USERS_C).Insert(user) + if err != nil { + return err + } + } + return nil +} + +func (m *GoMethodsMongoDbDriver) V001_init_users_down() error { + return m.Session.DB(DB_NAME).C(USERS_C).DropCollection() +} + +func (m *GoMethodsMongoDbDriver) V002_organizations_rename_location_field_to_headquarters_up() error { + c := m.Session.DB(DB_NAME).C(ORGANIZATIONS_C) + + _, err := c.UpdateAll(nil, bson.M{"$rename": bson.M{"location": "headquarters"}}) + return err +} + +func (m *GoMethodsMongoDbDriver) V002_organizations_rename_location_field_to_headquarters_down() error { + c := m.Session.DB(DB_NAME).C(ORGANIZATIONS_C) + + _, err := c.UpdateAll(nil, bson.M{"$rename": bson.M{"headquarters": "location"}}) + return err +} + +func (m *GoMethodsMongoDbDriver) V002_change_user_cleo_to_cleopatra_up() error { + c := m.Session.DB(DB_NAME).C(USERS_C) + + colQuerier := bson.M{"name": "Cleo"} + change := bson.M{"$set": bson.M{"name": "Cleopatra"}} + + return c.Update(colQuerier, change) +} + +func (m *GoMethodsMongoDbDriver) V002_change_user_cleo_to_cleopatra_down() error { + c := m.Session.DB(DB_NAME).C(USERS_C) + + colQuerier := bson.M{"name": "Cleopatra"} + change := bson.M{"$set": bson.M{"name": "Cleo",}} + + return c.Update(colQuerier, change) +} \ No newline at end of file diff --git a/driver/gomethods/usage_examples/mongodb_test.go b/driver/gomethods/usage_examples/mongodb_test.go new file mode 100644 index 00000000..ec1a85dd --- /dev/null +++ b/driver/gomethods/usage_examples/mongodb_test.go @@ -0,0 +1,123 @@ +package usage_examples + +import ( + "testing" + + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/migrate/direction" + + pipep "github.com/dimag-jfrog/migrate/pipe" +) + + + +func TestMigrate(t *testing.T) { + //host := os.Getenv("MONGODB_PORT_27017_TCP_ADDR") + //port := os.Getenv("MONGODB_PORT_27017_TCP_PORT") + host := "127.0.0.1" + port := "27017" + driverUrl := "mongodb://" + host + ":" + port + + d := &GoMethodsMongoDbDriver{} + if err := d.Initialize(driverUrl); err != nil { + t.Fatal(err) + } + + content1 := []byte(` + V001_init_organizations + V001_init_users + `) + content2 := []byte(` + V002_organizations_rename_location_field_to_headquarters + V002_change_user_cleo_to_cleopatra + `) + + files := []file.File{ + { + Path: "/foobar", + FileName: "001_foobar.mgo", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: content1, + }, + { + Path: "/foobar", + FileName: "001_foobar.mgo", + Version: 1, + Name: "foobar", + Direction: direction.Down, + Content: content1, + }, + { + Path: "/foobar", + FileName: "002_foobar.mgo", + Version: 2, + Name: "foobar", + Direction: direction.Up, + Content: content2, + }, + { + Path: "/foobar", + FileName: "002_foobar.mgo", + Version: 2, + Name: "foobar", + Direction: direction.Down, + Content: content2, + }, + { + Path: "/foobar", + FileName: "001_foobar.mgo", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V001_init_organizations + V001_init_users + V001_non_existing_operation + `), + }, + } + + var pipe chan interface{} + var errs []error + + pipe = pipep.New() + go d.Migrate(files[0], pipe) + errs = pipep.ReadErrors(pipe) + if len(errs) > 0 { + t.Fatal(errs) + } + + pipe = pipep.New() + go d.Migrate(files[2], pipe) + errs = pipep.ReadErrors(pipe) + if len(errs) > 0 { + t.Fatal(errs) + } + + pipe = pipep.New() + go d.Migrate(files[3], pipe) + errs = pipep.ReadErrors(pipe) + if len(errs) > 0 { + t.Fatal(errs) + } + + //pipe = pipep.New() + //go d.Migrate(files[1], pipe) + //errs = pipep.ReadErrors(pipe) + //if len(errs) > 0 { + // t.Fatal(errs) + //} + + pipe = pipep.New() + go d.Migrate(files[4], pipe) + errs = pipep.ReadErrors(pipe) + if len(errs) == 0 { + t.Error("Expected test case to fail") + } + + if err := d.Close(); err != nil { + t.Fatal(err) + } +} diff --git a/file/file.go b/file/file.go index 14617ba6..794e9515 100644 --- a/file/file.go +++ b/file/file.go @@ -9,20 +9,11 @@ import ( "go/token" "io/ioutil" "path" - "regexp" "sort" "strconv" "strings" ) -var filenameRegex = `^([0-9]+)_(.*)\.(up|down)\.%s$` - -// FilenameRegex builds regular expression stmt with given -// filename extension from driver. -func FilenameRegex(filenameExtension string) *regexp.Regexp { - return regexp.MustCompile(fmt.Sprintf(filenameRegex, filenameExtension)) -} - // File represents one file on disk. // Example: 001_initial_plan_to_do_sth.up.sql type File struct { @@ -149,8 +140,17 @@ func (mf *MigrationFiles) From(version uint64, relativeN int) (Files, error) { return files, nil } + +func ReadMigrationFiles(path, filenameExtension string) (MigrationFiles, error){ + return doReadMigrationFiles(path, DefaultFilenameParser{FilenameExtension: filenameExtension}) +} + +func ReadMigrationFilesWithFilenameParser(path string, filenameParser FilenameParser) (MigrationFiles, error){ + return doReadMigrationFiles(path, filenameParser) +} + // ReadMigrationFiles reads all migration files from a given path -func ReadMigrationFiles(path string, filenameRegex *regexp.Regexp) (files MigrationFiles, err error) { +func doReadMigrationFiles(path string, filenameParser FilenameParser) (files MigrationFiles, err error) { // find all migration files in path ioFiles, err := ioutil.ReadDir(path) if err != nil { @@ -165,7 +165,7 @@ func ReadMigrationFiles(path string, filenameRegex *regexp.Regexp) (files Migrat tmpFiles := make([]*tmpFile, 0) tmpFileMap := map[uint64]map[direction.Direction]tmpFile{} for _, file := range ioFiles { - version, name, d, err := parseFilenameSchema(file.Name(), filenameRegex) + version, name, d, err := filenameParser.Parse(file.Name()) if err == nil { if _, ok := tmpFileMap[version]; !ok { tmpFileMap[version] = map[direction.Direction]tmpFile{} @@ -210,33 +210,56 @@ func ReadMigrationFiles(path string, filenameRegex *regexp.Regexp) (files Migrat Direction: direction.Down, } lookFordirection = direction.Up + case direction.Both: + migrationFile.UpFile = &File{ + Path: path, + FileName: file.filename, + Version: file.version, + Name: file.name, + Content: nil, + Direction: direction.Up, + } + migrationFile.DownFile = &File{ + Path: path, + FileName: file.filename, + Version: file.version, + Name: file.name, + Content: nil, + Direction: direction.Down, + } default: return nil, errors.New("Unsupported direction.Direction Type") } for _, file2 := range tmpFiles { - if file2.version == file.version && file2.d == lookFordirection { - switch lookFordirection { - case direction.Up: - migrationFile.UpFile = &File{ - Path: path, - FileName: file2.filename, - Version: file.version, - Name: file2.name, - Content: nil, - Direction: direction.Up, + if file2.version == file.version { + if file.d == direction.Both { + if file.d != file2.d { + return nil, errors.New("Incompatible direction.Direction types") } - case direction.Down: - migrationFile.DownFile = &File{ - Path: path, - FileName: file2.filename, - Version: file.version, - Name: file2.name, - Content: nil, - Direction: direction.Down, + } else if file2.d == lookFordirection { + switch lookFordirection { + case direction.Up: + migrationFile.UpFile = &File{ + Path: path, + FileName: file2.filename, + Version: file.version, + Name: file2.name, + Content: nil, + Direction: direction.Up, + } + case direction.Down: + migrationFile.DownFile = &File{ + Path: path, + FileName: file2.filename, + Version: file.version, + Name: file2.name, + Content: nil, + Direction: direction.Down, + } } + break } - break } } @@ -249,29 +272,6 @@ func ReadMigrationFiles(path string, filenameRegex *regexp.Regexp) (files Migrat return newFiles, nil } -// parseFilenameSchema parses the filename -func parseFilenameSchema(filename string, filenameRegex *regexp.Regexp) (version uint64, name string, d direction.Direction, err error) { - matches := filenameRegex.FindStringSubmatch(filename) - if len(matches) != 4 { - return 0, "", 0, errors.New("Unable to parse filename schema") - } - - version, err = strconv.ParseUint(matches[1], 10, 0) - if err != nil { - return 0, "", 0, errors.New(fmt.Sprintf("Unable to parse version '%v' in filename schema", matches[0])) - } - - if matches[3] == "up" { - d = direction.Up - } else if matches[3] == "down" { - d = direction.Down - } else { - return 0, "", 0, errors.New(fmt.Sprintf("Unable to parse up|down '%v' in filename schema", matches[3])) - } - - return version, matches[2], d, nil -} - // Len is the number of elements in the collection. // Required by Sort Interface{} func (mf MigrationFiles) Len() int { diff --git a/file/file_test.go b/file/file_test.go index c6bd0b3c..648f5547 100644 --- a/file/file_test.go +++ b/file/file_test.go @@ -8,52 +8,6 @@ import ( "testing" ) -func TestParseFilenameSchema(t *testing.T) { - var tests = []struct { - filename string - filenameExtension string - expectVersion uint64 - expectName string - expectDirection direction.Direction - expectErr bool - }{ - {"001_test_file.up.sql", "sql", 1, "test_file", direction.Up, false}, - {"001_test_file.down.sql", "sql", 1, "test_file", direction.Down, false}, - {"10034_test_file.down.sql", "sql", 10034, "test_file", direction.Down, false}, - {"-1_test_file.down.sql", "sql", 0, "", direction.Up, true}, - {"test_file.down.sql", "sql", 0, "", direction.Up, true}, - {"100_test_file.down", "sql", 0, "", direction.Up, true}, - {"100_test_file.sql", "sql", 0, "", direction.Up, true}, - {"100_test_file", "sql", 0, "", direction.Up, true}, - {"test_file", "sql", 0, "", direction.Up, true}, - {"100", "sql", 0, "", direction.Up, true}, - {".sql", "sql", 0, "", direction.Up, true}, - {"up.sql", "sql", 0, "", direction.Up, true}, - {"down.sql", "sql", 0, "", direction.Up, true}, - } - - for _, test := range tests { - version, name, migrate, err := parseFilenameSchema(test.filename, FilenameRegex(test.filenameExtension)) - if test.expectErr && err == nil { - t.Fatal("Expected error, but got none.", test) - } - if !test.expectErr && err != nil { - t.Fatal("Did not expect error, but got one:", err, test) - } - if err == nil { - if version != test.expectVersion { - t.Error("Wrong version number", test) - } - if name != test.expectName { - t.Error("wrong name", test) - } - if migrate != test.expectDirection { - t.Error("wrong migrate", test) - } - } - } -} - func TestFiles(t *testing.T) { tmpdir, err := ioutil.TempDir("/tmp", "TestLookForMigrationFilesInSearchPath") if err != nil { @@ -77,7 +31,7 @@ func TestFiles(t *testing.T) { ioutil.WriteFile(path.Join(tmpdir, "401_migrationfile.down.sql"), []byte("test"), 0755) - files, err := ReadMigrationFiles(tmpdir, FilenameRegex("sql")) + files, err := ReadMigrationFiles(tmpdir, "sql") if err != nil { t.Fatal(err) } @@ -223,7 +177,7 @@ func TestDuplicateFiles(t *testing.T) { t.Fatal(err) } - _, err = ReadMigrationFiles(root, FilenameRegex("sql")) + _, err = ReadMigrationFiles(root, "sql") if err == nil { t.Fatal("Expected duplicate migration file error") } diff --git a/file/filename_parser.go b/file/filename_parser.go new file mode 100644 index 00000000..6fb85212 --- /dev/null +++ b/file/filename_parser.go @@ -0,0 +1,85 @@ +package file + +import ( + "errors" + "github.com/dimag-jfrog/migrate/migrate/direction" + "regexp" + "fmt" + "strconv" + "strings" +) + + +type FilenameParser interface { + Parse (filename string) (version uint64, name string, d direction.Direction, err error) +} + + +var defaultFilenameRegexTemplate = `^([0-9]+)_(.*)\.(up|down)\.%s$` + +func parseDefaultFilenameSchema(filename, filenameRegex string) (version uint64, name string, d direction.Direction, err error) { + regexp := regexp.MustCompile(filenameRegex) + matches := regexp.FindStringSubmatch(filename) + if len(matches) != 4 { + return 0, "", 0, errors.New("Unable to parse filename schema") + } + + version, err = strconv.ParseUint(matches[1], 10, 0) + if err != nil { + return 0, "", 0, errors.New(fmt.Sprintf("Unable to parse version '%v' in filename schema", matches[0])) + } + + name = matches[2] + + if matches[3] == "up" { + d = direction.Up + } else if matches[3] == "down" { + d = direction.Down + } else { + return 0, "", 0, errors.New(fmt.Sprintf("Unable to parse up|down '%v' in filename schema", matches[3])) + } + + return version, name, d, nil +} + +type DefaultFilenameParser struct { + FilenameExtension string +} + +func (parser DefaultFilenameParser) Parse (filename string) (version uint64, name string, d direction.Direction, err error) { + filenameRegex := fmt.Sprintf(defaultFilenameRegexTemplate, parser.FilenameExtension) + return parseDefaultFilenameSchema(filename, filenameRegex) +} + + +type UpDownAndBothFilenameParser struct { + FilenameExtension string +} + +func (parser UpDownAndBothFilenameParser) Parse(filename string) (version uint64, name string, d direction.Direction, err error) { + ext := parser.FilenameExtension + if !strings.HasSuffix(filename, ext) { + return 0, "", 0, errors.New("Filename ") + } + + var matches []string + if strings.HasSuffix(filename, ".up." + ext) || strings.HasSuffix(filename, ".down." + ext) { + filenameRegex := fmt.Sprintf(defaultFilenameRegexTemplate, parser.FilenameExtension) + return parseDefaultFilenameSchema(filename, filenameRegex) + } + + regex := regexp.MustCompile(fmt.Sprintf(`^([0-9]+)_(.*)\.%s$`, ext)) + matches = regex.FindStringSubmatch(filename) + if len(matches) != 3 { + return 0, "", 0, errors.New("Unable to parse filename schema") + } + + version, err = strconv.ParseUint(matches[1], 10, 0) + if err != nil { + return 0, "", 0, errors.New(fmt.Sprintf("Unable to parse version '%v' in filename schema", matches[0])) + } + name = matches[2] + d = direction.Both + + return version, name, d, nil +} diff --git a/file/filename_parser_test.go b/file/filename_parser_test.go new file mode 100644 index 00000000..ea9135d4 --- /dev/null +++ b/file/filename_parser_test.go @@ -0,0 +1,85 @@ +package file + +import ( + "github.com/dimag-jfrog/migrate/migrate/direction" + "testing" +) + + +type ParsingTest struct { + filename string + filenameExtension string + expectVersion uint64 + expectName string + expectDirection direction.Direction + expectErr bool +} + +func testParser(t *testing.T, parser FilenameParser, test *ParsingTest) { + version, name, migrate, err := parser.Parse(test.filename) + if test.expectErr && err == nil { + t.Fatal("Expected error, but got none.", test) + } + if !test.expectErr && err != nil { + t.Fatal("Did not expect error, but got one:", err, test) + } + if err == nil { + if version != test.expectVersion { + t.Error("Wrong version number", test) + } + if name != test.expectName { + t.Error("wrong name", test) + } + if migrate != test.expectDirection { + t.Error("wrong migrate", test) + } + } +} + +func TestParseDefaultFilenameSchema(t *testing.T) { + var tests = []ParsingTest { + {"001_test_file.up.sql", "sql", 1, "test_file", direction.Up, false}, + {"001_test_file.down.sql", "sql", 1, "test_file", direction.Down, false}, + {"10034_test_file.down.sql", "sql", 10034, "test_file", direction.Down, false}, + {"-1_test_file.down.sql", "sql", 0, "", direction.Up, true}, + {"test_file.down.sql", "sql", 0, "", direction.Up, true}, + {"100_test_file.down", "sql", 0, "", direction.Up, true}, + {"100_test_file.sql", "sql", 0, "", direction.Up, true}, + {"100_test_file", "sql", 0, "", direction.Up, true}, + {"test_file", "sql", 0, "", direction.Up, true}, + {"100", "sql", 0, "", direction.Up, true}, + {".sql", "sql", 0, "", direction.Up, true}, + {"up.sql", "sql", 0, "", direction.Up, true}, + {"down.sql", "sql", 0, "", direction.Up, true}, + } + + for _, test := range tests { + parser := DefaultFilenameParser{FilenameExtension:test.filenameExtension} + testParser(t, &parser, &test) + } +} + +func TestParseUpDownAndBothFilenameSchema(t *testing.T) { + var tests = []ParsingTest { + {"001_test_file.up.sql", "sql", 1, "test_file", direction.Up, false}, + {"001_test_file.down.sql", "sql", 1, "test_file", direction.Down, false}, + {"10034_test_file.down.sql", "sql", 10034, "test_file", direction.Down, false}, + {"-1_test_file.down.sql", "sql", 0, "", direction.Up, true}, + {"test_file.down.sql", "sql", 0, "", direction.Up, true}, + {"100_test_file.down", "sql", 0, "", direction.Up, true}, + {"100_test_file.sql", "sql", 100, "test_file", direction.Both, false}, + {"001_test_file.mgo", "mgo", 1, "test_file", direction.Both, false}, + {"-1_test_file.mgo", "sql", 0, "", direction.Up, true}, + {"100_test_file", "sql", 0, "", direction.Up, true}, + {"test_file", "sql", 0, "", direction.Up, true}, + {"100", "sql", 0, "", direction.Up, true}, + {".sql", "sql", 0, "", direction.Up, true}, + {"up.sql", "sql", 0, "", direction.Up, true}, + {"down.sql", "sql", 0, "", direction.Up, true}, + } + + for _, test := range tests { + parser := UpDownAndBothFilenameParser{FilenameExtension:test.filenameExtension} + testParser(t, &parser, &test) + } +} diff --git a/migrate/direction/direction.go b/migrate/direction/direction.go index ed5e5ece..3ebc7b8d 100644 --- a/migrate/direction/direction.go +++ b/migrate/direction/direction.go @@ -6,4 +6,5 @@ type Direction int const ( Up Direction = +1 Down = -1 + Both = 0 ) diff --git a/migrate/migrate.go b/migrate/migrate.go index 54cc42bc..4e3c53d7 100644 --- a/migrate/migrate.go +++ b/migrate/migrate.go @@ -212,7 +212,7 @@ func Create(url, migrationsPath, name string) (*file.MigrationFile, error) { if err != nil { return nil, err } - files, err := file.ReadMigrationFiles(migrationsPath, file.FilenameRegex(d.FilenameExtension())) + files, err := file.ReadMigrationFiles(migrationsPath, d.FilenameExtension()) if err != nil { return nil, err } @@ -268,7 +268,15 @@ func initDriverAndReadMigrationFilesAndGetVersion(url, migrationsPath string) (d if err != nil { return nil, nil, 0, err } - files, err := file.ReadMigrationFiles(migrationsPath, file.FilenameRegex(d.FilenameExtension())) + + var files file.MigrationFiles + + if d1, ok := d.(driver.DriverWithFilenameParser); ok { + files, err = file.ReadMigrationFilesWithFilenameParser(migrationsPath, d1.FilenameParser()) + } else { + files, err = file.ReadMigrationFiles(migrationsPath, d.FilenameExtension()) + } + if err != nil { d.Close() // TODO what happens with errors from this func? return nil, nil, 0, err From 5f7b2caee8518c70b5859f3c48e92e83a5340005 Mon Sep 17 00:00:00 2001 From: dimag Date: Mon, 8 Aug 2016 21:07:37 +0300 Subject: [PATCH 19/42] - Reset all changes to the upstream branch - Changed logic not to use custom filename parser: -Supporting up and down files only, no both direction files -Using method names as is - Added complete test to the mongo db migration scenarios --- .gitignore | 11 +- driver/driver.go | 9 - driver/gomethods/gomethods_migrator.go | 35 +-- driver/gomethods/gomethods_migrator_test.go | 95 ++----- driver/gomethods/mongodb/mongodb_template.go | 4 - driver/gomethods/usage_examples/mongodb.go | 31 +- .../gomethods/usage_examples/mongodb_test.go | 268 ++++++++++++------ file/file.go | 106 +++---- file/file_test.go | 50 +++- file/filename_parser.go | 85 ------ file/filename_parser_test.go | 85 ------ migrate/direction/direction.go | 1 - migrate/migrate.go | 12 +- 13 files changed, 361 insertions(+), 431 deletions(-) delete mode 100644 file/filename_parser.go delete mode 100644 file/filename_parser_test.go diff --git a/.gitignore b/.gitignore index 9bd9f472..ee2f41ff 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,2 @@ .DS_Store -test.db -*.iml -.DS_Store -.idea -*.zip -*.tar.* -*.nuget -*.jar -*.war - +test.db \ No newline at end of file diff --git a/driver/driver.go b/driver/driver.go index a668d52e..5c2f6fcc 100644 --- a/driver/driver.go +++ b/driver/driver.go @@ -34,15 +34,6 @@ type Driver interface { Version() (uint64, error) } -// Driver that has some custom migration file format -// and wants to use a different parsing strategy. -type DriverWithFilenameParser interface { - - Driver - - FilenameParser() file.FilenameParser -} - // New returns Driver and calls Initialize on it func New(url string) (Driver, error) { u, err := neturl.Parse(url) diff --git a/driver/gomethods/gomethods_migrator.go b/driver/gomethods/gomethods_migrator.go index 68b9ff52..d6953421 100644 --- a/driver/gomethods/gomethods_migrator.go +++ b/driver/gomethods/gomethods_migrator.go @@ -10,7 +10,6 @@ import ( "bufio" "github.com/dimag-jfrog/migrate/driver" "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/migrate/direction" ) @@ -32,7 +31,7 @@ func (e *MethodInvocationFailedError) Error() string { type Migrator struct { - Driver driver.DriverWithFilenameParser + Driver driver.Driver RollbackOnFailure bool } @@ -55,6 +54,10 @@ func (m *Migrator) Migrate(f file.File, pipe chan interface{}) error { // on failure, try to rollback methods in this migration for j := i-1; j >= 0; j-- { rollbackToMethodName := getRollbackToMethod(methods[j]) + if rollbackToMethodName == "" || !m.IsValid(rollbackToMethodName) { + continue + } + pipe <- rollbackToMethodName err = m.Invoke(rollbackToMethodName) if err != nil { @@ -106,8 +109,10 @@ func reverseInPlace(a []string) { func getRollbackToMethod(methodName string) string { if strings.HasSuffix(methodName, "_up") { return strings.TrimSuffix(methodName, "_up") + "_down" - } else { + } else if strings.HasSuffix(methodName, "_down") { return strings.TrimSuffix(methodName, "_down") + "_up" + } else { + return "" } } @@ -143,34 +148,20 @@ func (m *Migrator) getMigrationMethods(f file.File) ([]string, error) { } for _, line := range lines { - operationName := strings.TrimSpace(line) + methodName := strings.TrimSpace(line) - if operationName == "" || strings.HasPrefix(operationName, "--") { + if methodName == "" || strings.HasPrefix(methodName, "--") { // an empty line or a comment, ignore continue } - upMethodName := operationName + "_up" - downMethodName := operationName + "_down" - - if !m.IsValid(upMethodName) { - return nil, MissingMethodError(upMethodName) - } - if !m.IsValid(downMethodName) { - return nil, MissingMethodError(downMethodName) + if !m.IsValid(methodName) { + return nil, MissingMethodError(methodName) } - if f.Direction == direction.Up { - methods = append(methods, upMethodName) - } else { - methods = append(methods, downMethodName) - } + methods = append(methods, methodName) } - _,_,fileType,_ := m.Driver.FilenameParser().Parse(f.FileName) - if fileType == direction.Both && f.Direction == direction.Down { - reverseInPlace(methods) - } return methods, nil } diff --git a/driver/gomethods/gomethods_migrator_test.go b/driver/gomethods/gomethods_migrator_test.go index 58fd4f2b..1a5452f6 100644 --- a/driver/gomethods/gomethods_migrator_test.go +++ b/driver/gomethods/gomethods_migrator_test.go @@ -23,10 +23,6 @@ func (driver *FakeGoMethodsDriver) Close() error { return nil } -func (driver *FakeGoMethodsDriver) FilenameParser() file.FilenameParser { - return file.UpDownAndBothFilenameParser{ FilenameExtension: driver.FilenameExtension() } -} - func (driver *FakeGoMethodsDriver) FilenameExtension() string { return "gm" } @@ -84,39 +80,7 @@ func TestMigrate(t *testing.T) { expectRollback bool }{ { - name: "up migration, both directions-file: invokes up methods in order", - file: file.File { - Path: "/foobar", - FileName: "001_foobar.gm", - Version: 1, - Name: "foobar", - Direction: direction.Up, - Content: []byte(` - V001_init_organizations - V001_init_users - `), - }, - expectedInvokedMethods: []string{"V001_init_organizations_up", "V001_init_users_up"}, - expectedErrors: []error{}, - }, - { - name: "down migration, both-directions-file: reverts direction of invoked down methods", - file: file.File { - Path: "/foobar", - FileName: "001_foobar.gm", - Version: 1, - Name: "foobar", - Direction: direction.Down, - Content: []byte(` - V001_init_organizations - V001_init_users - `), - }, - expectedInvokedMethods: []string{"V001_init_users_down", "V001_init_organizations_down"}, - expectedErrors: []error{}, - }, - { - name: "up migration, up direction-file: invokes up methods in order", + name: "up migration invokes up methods", file: file.File { Path: "/foobar", FileName: "001_foobar.up.gm", @@ -124,15 +88,15 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Up, Content: []byte(` - V001_init_organizations - V001_init_users + V001_init_organizations_up + V001_init_users_up `), }, expectedInvokedMethods: []string{"V001_init_organizations_up", "V001_init_users_up"}, expectedErrors: []error{}, }, { - name: "down migration, down directions-file: keeps order of invoked down methods", + name: "down migration invoked down methods", file: file.File { Path: "/foobar", FileName: "001_foobar.down.gm", @@ -140,25 +104,25 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Down, Content: []byte(` - V001_init_organizations - V001_init_users + V001_init_users_down + V001_init_organizations_down `), }, - expectedInvokedMethods: []string{"V001_init_organizations_down", "V001_init_users_down"}, + expectedInvokedMethods: []string{"V001_init_users_down", "V001_init_organizations_down"}, expectedErrors: []error{}, }, { name: "up migration: non-existing method causes migration not to execute", file: file.File { Path: "/foobar", - FileName: "001_foobar.gm", + FileName: "001_foobar.up.gm", Version: 1, Name: "foobar", Direction: direction.Up, Content: []byte(` - V001_init_organizations - V001_init_users - V001_some_non_existing_method + V001_init_organizations_up + V001_init_users_up + V001_some_non_existing_method_up `), }, expectedInvokedMethods: []string{}, @@ -168,14 +132,14 @@ func TestMigrate(t *testing.T) { name: "up migration: failing method stops execution", file: file.File { Path: "/foobar", - FileName: "001_foobar.gm", + FileName: "001_foobar.up.gm", Version: 1, Name: "foobar", Direction: direction.Up, Content: []byte(` - V001_init_organizations - V001_some_failing_method - V001_init_users + V001_init_organizations_up + V001_some_failing_method_up + V001_init_users_up `), }, expectedInvokedMethods: []string{ @@ -188,17 +152,17 @@ func TestMigrate(t *testing.T) { }}, }, { - name: "down migration, both-directions-file: failing method stops migration", + name: "down migration: failing method stops migration", file: file.File { Path: "/foobar", - FileName: "001_foobar.gm", + FileName: "001_foobar.down.gm", Version: 1, Name: "foobar", Direction: direction.Down, Content: []byte(` - V001_init_organizations - V001_some_failing_method - V001_init_users + V001_init_users_down + V001_some_failing_method_down + V001_init_organizations_down `), }, expectedInvokedMethods: []string{ @@ -215,14 +179,14 @@ func TestMigrate(t *testing.T) { expectRollback: true, file: file.File { Path: "/foobar", - FileName: "001_foobar.gm", + FileName: "001_foobar.up.gm", Version: 1, Name: "foobar", Direction: direction.Up, Content: []byte(` - V001_init_organizations - V001_init_users - V001_some_failing_method + V001_init_organizations_up + V001_init_users_up + V001_some_failing_method_up `), }, expectedInvokedMethods: []string{ @@ -238,18 +202,18 @@ func TestMigrate(t *testing.T) { }}, }, { - name: "down migration, both-directions-file: failing method causes rollback in rollback mode", + name: "down migration: failing method causes rollback in rollback mode", expectRollback: true, file: file.File { Path: "/foobar", - FileName: "001_foobar.gm", + FileName: "001_foobar.down.gm", Version: 1, Name: "foobar", Direction: direction.Down, Content: []byte(` - V001_init_organizations - V001_some_failing_method - V001_init_users + V001_init_users_down + V001_some_failing_method_down + V001_init_organizations_down `), }, expectedInvokedMethods: []string{ @@ -306,6 +270,7 @@ func TestGetRollbackToMethod(t *testing.T) { {"up_down_up", "up_down_down"}, {"down_up", "down_down"}, {"down_down", "down_up"}, + {"some_method", ""}, } for _, c := range cases { diff --git a/driver/gomethods/mongodb/mongodb_template.go b/driver/gomethods/mongodb/mongodb_template.go index 48817cf6..1edbe78e 100644 --- a/driver/gomethods/mongodb/mongodb_template.go +++ b/driver/gomethods/mongodb/mongodb_template.go @@ -55,10 +55,6 @@ func (driver *DriverTemplate) Close() error { return nil } -func (driver *DriverTemplate) FilenameParser() file.FilenameParser { - return file.UpDownAndBothFilenameParser{FilenameExtension: driver.FilenameExtension()} -} - func (driver *DriverTemplate) FilenameExtension() string { return "mgo" } diff --git a/driver/gomethods/usage_examples/mongodb.go b/driver/gomethods/usage_examples/mongodb.go index be0ccb2d..ea7a680e 100644 --- a/driver/gomethods/usage_examples/mongodb.go +++ b/driver/gomethods/usage_examples/mongodb.go @@ -39,20 +39,39 @@ type Organization struct { DateFounded time.Time `bson:"date_founded"` } +type Organization_v2 struct { + Id bson.ObjectId `bson:"_id,omitempty"` + Name string `bson:"name"` + Headquarters string `bson:"headquarters"` + DateFounded time.Time `bson:"date_founded"` +} + type User struct { Id bson.ObjectId `bson:"_id"` Name string `bson:"name"` } +var OrganizationIds []bson.ObjectId = []bson.ObjectId{ + bson.NewObjectId(), + bson.NewObjectId(), + bson.NewObjectId(), +} + +var UserIds []bson.ObjectId = []bson.ObjectId{ + bson.NewObjectId(), + bson.NewObjectId(), + bson.NewObjectId(), +} + func (m *GoMethodsMongoDbDriver) V001_init_organizations_up() error { date1, _ := time.Parse(SHORT_DATE_LAYOUT, "1994-Jul-05") date2, _ := time.Parse(SHORT_DATE_LAYOUT, "1998-Sep-04") date3, _ := time.Parse(SHORT_DATE_LAYOUT, "2008-Apr-28") orgs := []Organization{ - {Id: bson.NewObjectId(), Name: "Amazon", Location:"Seattle", DateFounded: date1}, - {Id: bson.NewObjectId(), Name: "Google", Location:"Mountain View", DateFounded: date2}, - {Id: bson.NewObjectId(), Name: "JFrog", Location:"Santa Clara", DateFounded: date3}, + {Id: OrganizationIds[0], Name: "Amazon", Location:"Seattle", DateFounded: date1}, + {Id: OrganizationIds[1], Name: "Google", Location:"Mountain View", DateFounded: date2}, + {Id: OrganizationIds[2], Name: "JFrog", Location:"Santa Clara", DateFounded: date3}, } for _, org := range orgs { @@ -70,9 +89,9 @@ func (m *GoMethodsMongoDbDriver) V001_init_organizations_down() error { func (m *GoMethodsMongoDbDriver) V001_init_users_up() error { users := []User{ - {Id: bson.NewObjectId(), Name: "Alex"}, - {Id: bson.NewObjectId(), Name: "Beatrice"}, - {Id: bson.NewObjectId(), Name: "Cleo"}, + {Id: UserIds[0], Name: "Alex"}, + {Id: UserIds[1], Name: "Beatrice"}, + {Id: UserIds[2], Name: "Cleo"}, } for _, user := range users { diff --git a/driver/gomethods/usage_examples/mongodb_test.go b/driver/gomethods/usage_examples/mongodb_test.go index ec1a85dd..033876c8 100644 --- a/driver/gomethods/usage_examples/mongodb_test.go +++ b/driver/gomethods/usage_examples/mongodb_test.go @@ -7,9 +7,69 @@ import ( "github.com/dimag-jfrog/migrate/migrate/direction" pipep "github.com/dimag-jfrog/migrate/pipe" + "reflect" + "time" + "github.com/dimag-jfrog/migrate/driver/gomethods" ) +type ExpectedMigrationResult struct { + Organizations []Organization + Organizations_v2 []Organization_v2 + Users []User + Errors []error +} + +func RunMigrationAndAssertResult( + t *testing.T, + title string, + d *GoMethodsMongoDbDriver, + file file.File, + expected *ExpectedMigrationResult) { + + actualOrganizations := []Organization{} + actualOrganizations_v2 := []Organization_v2{} + actualUsers := []User{} + var err error + var pipe chan interface{} + var errs []error + + pipe = pipep.New() + go d.Migrate(file, pipe) + errs = pipep.ReadErrors(pipe) + + session := d.Session + if len(expected.Organizations) > 0 { + err = session.DB(DB_NAME).C(ORGANIZATIONS_C).Find(nil).All(&actualOrganizations) + } else { + err = session.DB(DB_NAME).C(ORGANIZATIONS_C).Find(nil).All(&actualOrganizations_v2) + } + if err != nil { + t.Fatal("Failed to query Organizations collection") + } + + err = session.DB(DB_NAME).C(USERS_C).Find(nil).All(&actualUsers) + if err != nil { + t.Fatal("Failed to query Users collection") + } + + if !reflect.DeepEqual(expected.Organizations, actualOrganizations) { + t.Fatalf("Migration '%s': FAILED\nexpected organizations %v\nbut got %v", title, expected.Organizations, actualOrganizations) + } + + if !reflect.DeepEqual(expected.Organizations_v2, actualOrganizations_v2) { + t.Fatalf("Migration '%s': FAILED\nexpected organizations v2 %v\nbut got %v", title, expected.Organizations_v2, actualOrganizations_v2) + } + + if !reflect.DeepEqual(expected.Users, actualUsers) { + t.Fatalf("Migration '%s': FAILED\nexpected users %v\nbut got %v", title, expected.Users, actualUsers) + + } + if !reflect.DeepEqual(expected.Errors, errs) { + t.Fatalf("Migration '%s': FAILED\nexpected errors %v\nbut got %v", title, expected.Errors, errs) + } +} + func TestMigrate(t *testing.T) { //host := os.Getenv("MONGODB_PORT_27017_TCP_ADDR") @@ -23,98 +83,148 @@ func TestMigrate(t *testing.T) { t.Fatal(err) } - content1 := []byte(` - V001_init_organizations - V001_init_users - `) - content2 := []byte(` - V002_organizations_rename_location_field_to_headquarters - V002_change_user_cleo_to_cleopatra - `) + // Reset DB + d.Session.DB(DB_NAME).C(ORGANIZATIONS_C).DropCollection() + d.Session.DB(DB_NAME).C(USERS_C).DropCollection() + + date1, _ := time.Parse(SHORT_DATE_LAYOUT, "1994-Jul-05") + date2, _ := time.Parse(SHORT_DATE_LAYOUT, "1998-Sep-04") + date3, _ := time.Parse(SHORT_DATE_LAYOUT, "2008-Apr-28") - files := []file.File{ + migrations := []struct { + name string + file file.File + expectedResult ExpectedMigrationResult + }{ { - Path: "/foobar", - FileName: "001_foobar.mgo", - Version: 1, - Name: "foobar", - Direction: direction.Up, - Content: content1, + name: "v0 -> v1", + file: file.File{ + Path: "/foobar", + FileName: "001_foobar.up.gm", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V001_init_organizations_up + V001_init_users_up + `), + }, + expectedResult: ExpectedMigrationResult{ + Organizations: []Organization{ + {Id: OrganizationIds[0], Name: "Amazon", Location:"Seattle", DateFounded: date1}, + {Id: OrganizationIds[1], Name: "Google", Location:"Mountain View", DateFounded: date2}, + {Id: OrganizationIds[2], Name: "JFrog", Location:"Santa Clara", DateFounded: date3}, + }, + Organizations_v2: []Organization_v2{}, + Users: []User{ + {Id: UserIds[0], Name: "Alex"}, + {Id: UserIds[1], Name: "Beatrice"}, + {Id: UserIds[2], Name: "Cleo"}, + }, + Errors: []error{}, + }, }, { - Path: "/foobar", - FileName: "001_foobar.mgo", - Version: 1, - Name: "foobar", - Direction: direction.Down, - Content: content1, + name: "v1 -> v2", + file: file.File{ + Path: "/foobar", + FileName: "002_foobar.up.gm", + Version: 2, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V002_organizations_rename_location_field_to_headquarters_up + V002_change_user_cleo_to_cleopatra_up + `), + }, + expectedResult: ExpectedMigrationResult{ + Organizations: []Organization{}, + Organizations_v2: []Organization_v2{ + {Id: OrganizationIds[0], Name: "Amazon", Headquarters:"Seattle", DateFounded: date1}, + {Id: OrganizationIds[1], Name: "Google", Headquarters:"Mountain View", DateFounded: date2}, + {Id: OrganizationIds[2], Name: "JFrog", Headquarters:"Santa Clara", DateFounded: date3}, + }, + Users: []User{ + {Id: UserIds[0], Name: "Alex"}, + {Id: UserIds[1], Name: "Beatrice"}, + {Id: UserIds[2], Name: "Cleopatra"}, + }, + Errors: []error{}, + }, }, { - Path: "/foobar", - FileName: "002_foobar.mgo", - Version: 2, - Name: "foobar", - Direction: direction.Up, - Content: content2, + name: "v2 -> v1", + file: file.File{ + Path: "/foobar", + FileName: "002_foobar.down.gm", + Version: 2, + Name: "foobar", + Direction: direction.Down, + Content: []byte(` + V002_change_user_cleo_to_cleopatra_down + V002_organizations_rename_location_field_to_headquarters_down + `), + }, + expectedResult: ExpectedMigrationResult{ + Organizations: []Organization{ + {Id: OrganizationIds[0], Name: "Amazon", Location:"Seattle", DateFounded: date1}, + {Id: OrganizationIds[1], Name: "Google", Location:"Mountain View", DateFounded: date2}, + {Id: OrganizationIds[2], Name: "JFrog", Location:"Santa Clara", DateFounded: date3}, + }, + Organizations_v2: []Organization_v2{}, + Users: []User{ + {Id: UserIds[0], Name: "Alex"}, + {Id: UserIds[1], Name: "Beatrice"}, + {Id: UserIds[2], Name: "Cleo"}, + }, + Errors: []error{}, + }, }, { - Path: "/foobar", - FileName: "002_foobar.mgo", - Version: 2, - Name: "foobar", - Direction: direction.Down, - Content: content2, + name: "v1 -> v0", + file: file.File{ + Path: "/foobar", + FileName: "001_foobar.down.gm", + Version: 1, + Name: "foobar", + Direction: direction.Down, + Content: []byte(` + V001_init_users_down + V001_init_organizations_down + `), + }, + expectedResult: ExpectedMigrationResult{ + Organizations: []Organization{}, + Organizations_v2: []Organization_v2{}, + Users: []User{}, + Errors: []error{}, + }, }, { - Path: "/foobar", - FileName: "001_foobar.mgo", - Version: 1, - Name: "foobar", - Direction: direction.Up, - Content: []byte(` - V001_init_organizations - V001_init_users - V001_non_existing_operation - `), + name: "v0 -> v1: with error", + file: file.File{ + Path: "/foobar", + FileName: "001_foobar.up.gm", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V001_init_organizations_up + V001_init_users_up + v001_non_existing_method_up + `), + }, + expectedResult: ExpectedMigrationResult{ + Organizations: []Organization{}, + Organizations_v2: []Organization_v2{}, + Users: []User{}, + Errors: []error{ gomethods.MissingMethodError("v001_non_existing_method_up") }, + }, }, } - var pipe chan interface{} - var errs []error - - pipe = pipep.New() - go d.Migrate(files[0], pipe) - errs = pipep.ReadErrors(pipe) - if len(errs) > 0 { - t.Fatal(errs) - } - - pipe = pipep.New() - go d.Migrate(files[2], pipe) - errs = pipep.ReadErrors(pipe) - if len(errs) > 0 { - t.Fatal(errs) - } - - pipe = pipep.New() - go d.Migrate(files[3], pipe) - errs = pipep.ReadErrors(pipe) - if len(errs) > 0 { - t.Fatal(errs) - } - - //pipe = pipep.New() - //go d.Migrate(files[1], pipe) - //errs = pipep.ReadErrors(pipe) - //if len(errs) > 0 { - // t.Fatal(errs) - //} - - pipe = pipep.New() - go d.Migrate(files[4], pipe) - errs = pipep.ReadErrors(pipe) - if len(errs) == 0 { - t.Error("Expected test case to fail") + for _, m := range migrations { + RunMigrationAndAssertResult(t, m.name, d, m.file, &m.expectedResult) } if err := d.Close(); err != nil { diff --git a/file/file.go b/file/file.go index 794e9515..14617ba6 100644 --- a/file/file.go +++ b/file/file.go @@ -9,11 +9,20 @@ import ( "go/token" "io/ioutil" "path" + "regexp" "sort" "strconv" "strings" ) +var filenameRegex = `^([0-9]+)_(.*)\.(up|down)\.%s$` + +// FilenameRegex builds regular expression stmt with given +// filename extension from driver. +func FilenameRegex(filenameExtension string) *regexp.Regexp { + return regexp.MustCompile(fmt.Sprintf(filenameRegex, filenameExtension)) +} + // File represents one file on disk. // Example: 001_initial_plan_to_do_sth.up.sql type File struct { @@ -140,17 +149,8 @@ func (mf *MigrationFiles) From(version uint64, relativeN int) (Files, error) { return files, nil } - -func ReadMigrationFiles(path, filenameExtension string) (MigrationFiles, error){ - return doReadMigrationFiles(path, DefaultFilenameParser{FilenameExtension: filenameExtension}) -} - -func ReadMigrationFilesWithFilenameParser(path string, filenameParser FilenameParser) (MigrationFiles, error){ - return doReadMigrationFiles(path, filenameParser) -} - // ReadMigrationFiles reads all migration files from a given path -func doReadMigrationFiles(path string, filenameParser FilenameParser) (files MigrationFiles, err error) { +func ReadMigrationFiles(path string, filenameRegex *regexp.Regexp) (files MigrationFiles, err error) { // find all migration files in path ioFiles, err := ioutil.ReadDir(path) if err != nil { @@ -165,7 +165,7 @@ func doReadMigrationFiles(path string, filenameParser FilenameParser) (files Mig tmpFiles := make([]*tmpFile, 0) tmpFileMap := map[uint64]map[direction.Direction]tmpFile{} for _, file := range ioFiles { - version, name, d, err := filenameParser.Parse(file.Name()) + version, name, d, err := parseFilenameSchema(file.Name(), filenameRegex) if err == nil { if _, ok := tmpFileMap[version]; !ok { tmpFileMap[version] = map[direction.Direction]tmpFile{} @@ -210,56 +210,33 @@ func doReadMigrationFiles(path string, filenameParser FilenameParser) (files Mig Direction: direction.Down, } lookFordirection = direction.Up - case direction.Both: - migrationFile.UpFile = &File{ - Path: path, - FileName: file.filename, - Version: file.version, - Name: file.name, - Content: nil, - Direction: direction.Up, - } - migrationFile.DownFile = &File{ - Path: path, - FileName: file.filename, - Version: file.version, - Name: file.name, - Content: nil, - Direction: direction.Down, - } default: return nil, errors.New("Unsupported direction.Direction Type") } for _, file2 := range tmpFiles { - if file2.version == file.version { - if file.d == direction.Both { - if file.d != file2.d { - return nil, errors.New("Incompatible direction.Direction types") + if file2.version == file.version && file2.d == lookFordirection { + switch lookFordirection { + case direction.Up: + migrationFile.UpFile = &File{ + Path: path, + FileName: file2.filename, + Version: file.version, + Name: file2.name, + Content: nil, + Direction: direction.Up, } - } else if file2.d == lookFordirection { - switch lookFordirection { - case direction.Up: - migrationFile.UpFile = &File{ - Path: path, - FileName: file2.filename, - Version: file.version, - Name: file2.name, - Content: nil, - Direction: direction.Up, - } - case direction.Down: - migrationFile.DownFile = &File{ - Path: path, - FileName: file2.filename, - Version: file.version, - Name: file2.name, - Content: nil, - Direction: direction.Down, - } + case direction.Down: + migrationFile.DownFile = &File{ + Path: path, + FileName: file2.filename, + Version: file.version, + Name: file2.name, + Content: nil, + Direction: direction.Down, } - break } + break } } @@ -272,6 +249,29 @@ func doReadMigrationFiles(path string, filenameParser FilenameParser) (files Mig return newFiles, nil } +// parseFilenameSchema parses the filename +func parseFilenameSchema(filename string, filenameRegex *regexp.Regexp) (version uint64, name string, d direction.Direction, err error) { + matches := filenameRegex.FindStringSubmatch(filename) + if len(matches) != 4 { + return 0, "", 0, errors.New("Unable to parse filename schema") + } + + version, err = strconv.ParseUint(matches[1], 10, 0) + if err != nil { + return 0, "", 0, errors.New(fmt.Sprintf("Unable to parse version '%v' in filename schema", matches[0])) + } + + if matches[3] == "up" { + d = direction.Up + } else if matches[3] == "down" { + d = direction.Down + } else { + return 0, "", 0, errors.New(fmt.Sprintf("Unable to parse up|down '%v' in filename schema", matches[3])) + } + + return version, matches[2], d, nil +} + // Len is the number of elements in the collection. // Required by Sort Interface{} func (mf MigrationFiles) Len() int { diff --git a/file/file_test.go b/file/file_test.go index 648f5547..c6bd0b3c 100644 --- a/file/file_test.go +++ b/file/file_test.go @@ -8,6 +8,52 @@ import ( "testing" ) +func TestParseFilenameSchema(t *testing.T) { + var tests = []struct { + filename string + filenameExtension string + expectVersion uint64 + expectName string + expectDirection direction.Direction + expectErr bool + }{ + {"001_test_file.up.sql", "sql", 1, "test_file", direction.Up, false}, + {"001_test_file.down.sql", "sql", 1, "test_file", direction.Down, false}, + {"10034_test_file.down.sql", "sql", 10034, "test_file", direction.Down, false}, + {"-1_test_file.down.sql", "sql", 0, "", direction.Up, true}, + {"test_file.down.sql", "sql", 0, "", direction.Up, true}, + {"100_test_file.down", "sql", 0, "", direction.Up, true}, + {"100_test_file.sql", "sql", 0, "", direction.Up, true}, + {"100_test_file", "sql", 0, "", direction.Up, true}, + {"test_file", "sql", 0, "", direction.Up, true}, + {"100", "sql", 0, "", direction.Up, true}, + {".sql", "sql", 0, "", direction.Up, true}, + {"up.sql", "sql", 0, "", direction.Up, true}, + {"down.sql", "sql", 0, "", direction.Up, true}, + } + + for _, test := range tests { + version, name, migrate, err := parseFilenameSchema(test.filename, FilenameRegex(test.filenameExtension)) + if test.expectErr && err == nil { + t.Fatal("Expected error, but got none.", test) + } + if !test.expectErr && err != nil { + t.Fatal("Did not expect error, but got one:", err, test) + } + if err == nil { + if version != test.expectVersion { + t.Error("Wrong version number", test) + } + if name != test.expectName { + t.Error("wrong name", test) + } + if migrate != test.expectDirection { + t.Error("wrong migrate", test) + } + } + } +} + func TestFiles(t *testing.T) { tmpdir, err := ioutil.TempDir("/tmp", "TestLookForMigrationFilesInSearchPath") if err != nil { @@ -31,7 +77,7 @@ func TestFiles(t *testing.T) { ioutil.WriteFile(path.Join(tmpdir, "401_migrationfile.down.sql"), []byte("test"), 0755) - files, err := ReadMigrationFiles(tmpdir, "sql") + files, err := ReadMigrationFiles(tmpdir, FilenameRegex("sql")) if err != nil { t.Fatal(err) } @@ -177,7 +223,7 @@ func TestDuplicateFiles(t *testing.T) { t.Fatal(err) } - _, err = ReadMigrationFiles(root, "sql") + _, err = ReadMigrationFiles(root, FilenameRegex("sql")) if err == nil { t.Fatal("Expected duplicate migration file error") } diff --git a/file/filename_parser.go b/file/filename_parser.go deleted file mode 100644 index 6fb85212..00000000 --- a/file/filename_parser.go +++ /dev/null @@ -1,85 +0,0 @@ -package file - -import ( - "errors" - "github.com/dimag-jfrog/migrate/migrate/direction" - "regexp" - "fmt" - "strconv" - "strings" -) - - -type FilenameParser interface { - Parse (filename string) (version uint64, name string, d direction.Direction, err error) -} - - -var defaultFilenameRegexTemplate = `^([0-9]+)_(.*)\.(up|down)\.%s$` - -func parseDefaultFilenameSchema(filename, filenameRegex string) (version uint64, name string, d direction.Direction, err error) { - regexp := regexp.MustCompile(filenameRegex) - matches := regexp.FindStringSubmatch(filename) - if len(matches) != 4 { - return 0, "", 0, errors.New("Unable to parse filename schema") - } - - version, err = strconv.ParseUint(matches[1], 10, 0) - if err != nil { - return 0, "", 0, errors.New(fmt.Sprintf("Unable to parse version '%v' in filename schema", matches[0])) - } - - name = matches[2] - - if matches[3] == "up" { - d = direction.Up - } else if matches[3] == "down" { - d = direction.Down - } else { - return 0, "", 0, errors.New(fmt.Sprintf("Unable to parse up|down '%v' in filename schema", matches[3])) - } - - return version, name, d, nil -} - -type DefaultFilenameParser struct { - FilenameExtension string -} - -func (parser DefaultFilenameParser) Parse (filename string) (version uint64, name string, d direction.Direction, err error) { - filenameRegex := fmt.Sprintf(defaultFilenameRegexTemplate, parser.FilenameExtension) - return parseDefaultFilenameSchema(filename, filenameRegex) -} - - -type UpDownAndBothFilenameParser struct { - FilenameExtension string -} - -func (parser UpDownAndBothFilenameParser) Parse(filename string) (version uint64, name string, d direction.Direction, err error) { - ext := parser.FilenameExtension - if !strings.HasSuffix(filename, ext) { - return 0, "", 0, errors.New("Filename ") - } - - var matches []string - if strings.HasSuffix(filename, ".up." + ext) || strings.HasSuffix(filename, ".down." + ext) { - filenameRegex := fmt.Sprintf(defaultFilenameRegexTemplate, parser.FilenameExtension) - return parseDefaultFilenameSchema(filename, filenameRegex) - } - - regex := regexp.MustCompile(fmt.Sprintf(`^([0-9]+)_(.*)\.%s$`, ext)) - matches = regex.FindStringSubmatch(filename) - if len(matches) != 3 { - return 0, "", 0, errors.New("Unable to parse filename schema") - } - - version, err = strconv.ParseUint(matches[1], 10, 0) - if err != nil { - return 0, "", 0, errors.New(fmt.Sprintf("Unable to parse version '%v' in filename schema", matches[0])) - } - name = matches[2] - d = direction.Both - - return version, name, d, nil -} diff --git a/file/filename_parser_test.go b/file/filename_parser_test.go deleted file mode 100644 index ea9135d4..00000000 --- a/file/filename_parser_test.go +++ /dev/null @@ -1,85 +0,0 @@ -package file - -import ( - "github.com/dimag-jfrog/migrate/migrate/direction" - "testing" -) - - -type ParsingTest struct { - filename string - filenameExtension string - expectVersion uint64 - expectName string - expectDirection direction.Direction - expectErr bool -} - -func testParser(t *testing.T, parser FilenameParser, test *ParsingTest) { - version, name, migrate, err := parser.Parse(test.filename) - if test.expectErr && err == nil { - t.Fatal("Expected error, but got none.", test) - } - if !test.expectErr && err != nil { - t.Fatal("Did not expect error, but got one:", err, test) - } - if err == nil { - if version != test.expectVersion { - t.Error("Wrong version number", test) - } - if name != test.expectName { - t.Error("wrong name", test) - } - if migrate != test.expectDirection { - t.Error("wrong migrate", test) - } - } -} - -func TestParseDefaultFilenameSchema(t *testing.T) { - var tests = []ParsingTest { - {"001_test_file.up.sql", "sql", 1, "test_file", direction.Up, false}, - {"001_test_file.down.sql", "sql", 1, "test_file", direction.Down, false}, - {"10034_test_file.down.sql", "sql", 10034, "test_file", direction.Down, false}, - {"-1_test_file.down.sql", "sql", 0, "", direction.Up, true}, - {"test_file.down.sql", "sql", 0, "", direction.Up, true}, - {"100_test_file.down", "sql", 0, "", direction.Up, true}, - {"100_test_file.sql", "sql", 0, "", direction.Up, true}, - {"100_test_file", "sql", 0, "", direction.Up, true}, - {"test_file", "sql", 0, "", direction.Up, true}, - {"100", "sql", 0, "", direction.Up, true}, - {".sql", "sql", 0, "", direction.Up, true}, - {"up.sql", "sql", 0, "", direction.Up, true}, - {"down.sql", "sql", 0, "", direction.Up, true}, - } - - for _, test := range tests { - parser := DefaultFilenameParser{FilenameExtension:test.filenameExtension} - testParser(t, &parser, &test) - } -} - -func TestParseUpDownAndBothFilenameSchema(t *testing.T) { - var tests = []ParsingTest { - {"001_test_file.up.sql", "sql", 1, "test_file", direction.Up, false}, - {"001_test_file.down.sql", "sql", 1, "test_file", direction.Down, false}, - {"10034_test_file.down.sql", "sql", 10034, "test_file", direction.Down, false}, - {"-1_test_file.down.sql", "sql", 0, "", direction.Up, true}, - {"test_file.down.sql", "sql", 0, "", direction.Up, true}, - {"100_test_file.down", "sql", 0, "", direction.Up, true}, - {"100_test_file.sql", "sql", 100, "test_file", direction.Both, false}, - {"001_test_file.mgo", "mgo", 1, "test_file", direction.Both, false}, - {"-1_test_file.mgo", "sql", 0, "", direction.Up, true}, - {"100_test_file", "sql", 0, "", direction.Up, true}, - {"test_file", "sql", 0, "", direction.Up, true}, - {"100", "sql", 0, "", direction.Up, true}, - {".sql", "sql", 0, "", direction.Up, true}, - {"up.sql", "sql", 0, "", direction.Up, true}, - {"down.sql", "sql", 0, "", direction.Up, true}, - } - - for _, test := range tests { - parser := UpDownAndBothFilenameParser{FilenameExtension:test.filenameExtension} - testParser(t, &parser, &test) - } -} diff --git a/migrate/direction/direction.go b/migrate/direction/direction.go index 3ebc7b8d..ed5e5ece 100644 --- a/migrate/direction/direction.go +++ b/migrate/direction/direction.go @@ -6,5 +6,4 @@ type Direction int const ( Up Direction = +1 Down = -1 - Both = 0 ) diff --git a/migrate/migrate.go b/migrate/migrate.go index 4e3c53d7..54cc42bc 100644 --- a/migrate/migrate.go +++ b/migrate/migrate.go @@ -212,7 +212,7 @@ func Create(url, migrationsPath, name string) (*file.MigrationFile, error) { if err != nil { return nil, err } - files, err := file.ReadMigrationFiles(migrationsPath, d.FilenameExtension()) + files, err := file.ReadMigrationFiles(migrationsPath, file.FilenameRegex(d.FilenameExtension())) if err != nil { return nil, err } @@ -268,15 +268,7 @@ func initDriverAndReadMigrationFilesAndGetVersion(url, migrationsPath string) (d if err != nil { return nil, nil, 0, err } - - var files file.MigrationFiles - - if d1, ok := d.(driver.DriverWithFilenameParser); ok { - files, err = file.ReadMigrationFilesWithFilenameParser(migrationsPath, d1.FilenameParser()) - } else { - files, err = file.ReadMigrationFiles(migrationsPath, d.FilenameExtension()) - } - + files, err := file.ReadMigrationFiles(migrationsPath, file.FilenameRegex(d.FilenameExtension())) if err != nil { d.Close() // TODO what happens with errors from this func? return nil, nil, 0, err From c71fea90d66a48ce091b694c02bb0c15b2248582 Mon Sep 17 00:00:00 2001 From: dimag Date: Mon, 8 Aug 2016 21:34:52 +0300 Subject: [PATCH 20/42] Relaxed constratint that method receiver has to be of Driver interface. --- driver/gomethods/gomethods_migrator.go | 7 +++---- driver/gomethods/gomethods_migrator_test.go | 2 +- driver/gomethods/usage_examples/mongodb.go | 2 +- driver/gomethods/usage_examples/mongodb_test.go | 1 + 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/driver/gomethods/gomethods_migrator.go b/driver/gomethods/gomethods_migrator.go index d6953421..0ff848a6 100644 --- a/driver/gomethods/gomethods_migrator.go +++ b/driver/gomethods/gomethods_migrator.go @@ -8,7 +8,6 @@ import ( "os" "path" "bufio" - "github.com/dimag-jfrog/migrate/driver" "github.com/dimag-jfrog/migrate/file" ) @@ -31,7 +30,7 @@ func (e *MethodInvocationFailedError) Error() string { type Migrator struct { - Driver driver.Driver + MigrationMethodsReceiver interface{} RollbackOnFailure bool } @@ -73,12 +72,12 @@ func (m *Migrator) Migrate(f file.File, pipe chan interface{}) error { } func (m *Migrator) IsValid(methodName string) bool { - return reflect.ValueOf(m.Driver).MethodByName(methodName).IsValid() + return reflect.ValueOf(m.MigrationMethodsReceiver).MethodByName(methodName).IsValid() } func (m *Migrator) Invoke(methodName string) error { name := methodName - migrateMethod := reflect.ValueOf(m.Driver).MethodByName(name) + migrateMethod := reflect.ValueOf(m.MigrationMethodsReceiver).MethodByName(name) if !migrateMethod.IsValid() { return MissingMethodError(methodName) } diff --git a/driver/gomethods/gomethods_migrator_test.go b/driver/gomethods/gomethods_migrator_test.go index 1a5452f6..a50d6c1d 100644 --- a/driver/gomethods/gomethods_migrator_test.go +++ b/driver/gomethods/gomethods_migrator_test.go @@ -232,7 +232,7 @@ func TestMigrate(t *testing.T) { for _, c := range cases { migrator := Migrator{} d := &FakeGoMethodsDriver{Migrator: migrator, InvokedMethods:[]string{}} - migrator.Driver = d + migrator.MigrationMethodsReceiver = d migrator.RollbackOnFailure = c.expectRollback pipe := pipep.New() diff --git a/driver/gomethods/usage_examples/mongodb.go b/driver/gomethods/usage_examples/mongodb.go index ea7a680e..e7f30c0a 100644 --- a/driver/gomethods/usage_examples/mongodb.go +++ b/driver/gomethods/usage_examples/mongodb.go @@ -16,7 +16,7 @@ type GoMethodsMongoDbDriver struct { } func (d *GoMethodsMongoDbDriver) Initialize(url string) error { - return d.DriverTemplate.Initialize(url, DB_NAME, gomethods.Migrator{Driver: d}) + return d.DriverTemplate.Initialize(url, DB_NAME, gomethods.Migrator{MigrationMethodsReceiver: d}) } func init() { diff --git a/driver/gomethods/usage_examples/mongodb_test.go b/driver/gomethods/usage_examples/mongodb_test.go index 033876c8..a57c51da 100644 --- a/driver/gomethods/usage_examples/mongodb_test.go +++ b/driver/gomethods/usage_examples/mongodb_test.go @@ -68,6 +68,7 @@ func RunMigrationAndAssertResult( if !reflect.DeepEqual(expected.Errors, errs) { t.Fatalf("Migration '%s': FAILED\nexpected errors %v\nbut got %v", title, expected.Errors, errs) } + t.Logf("Migration '%s': PASSED", title) } From 570e4b42be4f85e573e605e6885bfa64951a49f2 Mon Sep 17 00:00:00 2001 From: dimag Date: Tue, 9 Aug 2016 16:47:34 +0300 Subject: [PATCH 21/42] - redefined mongo db driver to be autonomous and not depending on specific methods - defined methods receivers registration by name and change the migration files format to include them - added extensive testing --- driver/gomethods/gomethods_migrator.go | 105 ++++++------- driver/gomethods/gomethods_migrator_test.go | 142 +++++++----------- driver/gomethods/gomethods_registry.go | 30 ++++ driver/gomethods/mongodb/mongodb.go | 134 +++++++++++++++++ driver/gomethods/mongodb/mongodb_template.go | 107 ------------- .../mongodb_example/methods_receiver.go | 129 ++++++++++++++++ .../mongodb_test.go | 85 +++++++---- driver/gomethods/usage_examples/mongodb.go | 140 ----------------- 8 files changed, 461 insertions(+), 411 deletions(-) create mode 100644 driver/gomethods/gomethods_registry.go create mode 100644 driver/gomethods/mongodb/mongodb.go delete mode 100644 driver/gomethods/mongodb/mongodb_template.go create mode 100644 driver/gomethods/mongodb_example/methods_receiver.go rename driver/gomethods/{usage_examples => mongodb_example}/mongodb_test.go (71%) delete mode 100644 driver/gomethods/usage_examples/mongodb.go diff --git a/driver/gomethods/gomethods_migrator.go b/driver/gomethods/gomethods_migrator.go index 0ff848a6..2ee01925 100644 --- a/driver/gomethods/gomethods_migrator.go +++ b/driver/gomethods/gomethods_migrator.go @@ -2,40 +2,57 @@ package gomethods import ( //"bytes" - "reflect" + "bufio" + "database/sql/driver" "fmt" - "strings" + "github.com/dimag-jfrog/migrate/file" "os" "path" - "bufio" - "github.com/dimag-jfrog/migrate/file" + "strings" ) +type UnregisteredMethodsReceiverError string + +func (e UnregisteredMethodsReceiverError) Error() string { + return "Unregistered methods receiver: " + string(e) +} type MissingMethodError string -func (e MissingMethodError) Error() string { return "Non existing migrate method: " + string(e) } +func (e MissingMethodError) Error() string { return "Non existing migrate method: " + string(e) } type WrongMethodSignatureError string -func (e WrongMethodSignatureError) Error() string { return fmt.Sprintf("Method %s has wrong signature", e) } + +func (e WrongMethodSignatureError) Error() string { + return fmt.Sprintf("Method %s has wrong signature", e) +} type MethodInvocationFailedError struct { MethodName string - Err error + Err error } func (e *MethodInvocationFailedError) Error() string { return fmt.Sprintf("Method %s returned an error: %v", e.MethodName, e.Error) } +type MigrationMethodInvoker interface { + IsValid(methodName string, methodReceiver interface{}) bool + Invoke(methodName string, methodReceiver interface{}) error +} + +type GoMethodsDriver interface { + driver.Driver + MigrationMethodInvoker +} type Migrator struct { - MigrationMethodsReceiver interface{} RollbackOnFailure bool + MethodInvoker MigrationMethodInvoker } func (m *Migrator) Migrate(f file.File, pipe chan interface{}) error { - methods, err := m.getMigrationMethods(f) + methods, methodsReceiver, err := m.getMigrationMethods(f) if err != nil { pipe <- err return err @@ -43,7 +60,7 @@ func (m *Migrator) Migrate(f file.File, pipe chan interface{}) error { for i, methodName := range methods { pipe <- methodName - err := m.Invoke(methodName) + err := m.MethodInvoker.Invoke(methodName, methodsReceiver) if err != nil { pipe <- err if !m.RollbackOnFailure { @@ -51,14 +68,15 @@ func (m *Migrator) Migrate(f file.File, pipe chan interface{}) error { } // on failure, try to rollback methods in this migration - for j := i-1; j >= 0; j-- { + for j := i - 1; j >= 0; j-- { rollbackToMethodName := getRollbackToMethod(methods[j]) - if rollbackToMethodName == "" || !m.IsValid(rollbackToMethodName) { + if rollbackToMethodName == "" || + !m.MethodInvoker.IsValid(rollbackToMethodName, methodsReceiver) { continue } pipe <- rollbackToMethodName - err = m.Invoke(rollbackToMethodName) + err = m.MethodInvoker.Invoke(rollbackToMethodName, methodsReceiver) if err != nil { pipe <- err break @@ -71,33 +89,6 @@ func (m *Migrator) Migrate(f file.File, pipe chan interface{}) error { return nil } -func (m *Migrator) IsValid(methodName string) bool { - return reflect.ValueOf(m.MigrationMethodsReceiver).MethodByName(methodName).IsValid() -} - -func (m *Migrator) Invoke(methodName string) error { - name := methodName - migrateMethod := reflect.ValueOf(m.MigrationMethodsReceiver).MethodByName(name) - if !migrateMethod.IsValid() { - return MissingMethodError(methodName) - } - - retValues := migrateMethod.Call([]reflect.Value{}) - if len(retValues) != 1 { - return WrongMethodSignatureError(name) - } - - if !retValues[0].IsNil() { - err, ok := retValues[0].Interface().(error) - if !ok { - return WrongMethodSignatureError(name) - } - return &MethodInvocationFailedError{ MethodName:name, Err:err} - } - - return nil -} - func reverseInPlace(a []string) { for i := 0; i < len(a)/2; i++ { j := len(a) - i - 1 @@ -139,28 +130,40 @@ func getFileLines(file file.File) ([]string, error) { } } -func (m *Migrator) getMigrationMethods(f file.File) ([]string, error) { - var lines, methods []string - lines, err := getFileLines(f) +func (m *Migrator) getMigrationMethods(f file.File) (methods []string, methodsReceiver interface{}, err error) { + var lines []string + + lines, err = getFileLines(f) if err != nil { - return nil, err + return nil, nil, err } for _, line := range lines { - methodName := strings.TrimSpace(line) + line := strings.TrimSpace(line) - if methodName == "" || strings.HasPrefix(methodName, "--") { + if line == "" || strings.HasPrefix(line, "--") { // an empty line or a comment, ignore continue } - if !m.IsValid(methodName) { - return nil, MissingMethodError(methodName) - } + if methodsReceiver == nil { + receiverName := line + methodsReceiver = GetMethodsReceiver(receiverName) + if methodsReceiver == nil { + return nil, nil, UnregisteredMethodsReceiverError(receiverName) + } + continue + + } else { + methodName := line + if !m.MethodInvoker.IsValid(methodName, methodsReceiver) { + return nil, nil, MissingMethodError(methodName) + } - methods = append(methods, methodName) + methods = append(methods, methodName) + } } - return methods, nil + return methods, methodsReceiver, nil } diff --git a/driver/gomethods/gomethods_migrator_test.go b/driver/gomethods/gomethods_migrator_test.go index a50d6c1d..eb33103f 100644 --- a/driver/gomethods/gomethods_migrator_test.go +++ b/driver/gomethods/gomethods_migrator_test.go @@ -10,133 +10,105 @@ import ( pipep "github.com/dimag-jfrog/migrate/pipe" ) -type FakeGoMethodsDriver struct { +type FakeGoMethodsInvoker struct { InvokedMethods []string - Migrator Migrator } -func (driver *FakeGoMethodsDriver) Initialize(url string) error { - return nil -} - -func (driver *FakeGoMethodsDriver) Close() error { - return nil -} - -func (driver *FakeGoMethodsDriver) FilenameExtension() string { - return "gm" -} - -func (driver *FakeGoMethodsDriver) Version() (uint64, error) { - return uint64(0), nil -} - -func (driver *FakeGoMethodsDriver) Migrate(f file.File, pipe chan interface{}) { - defer close(pipe) - pipe <- f - return -} +func (invoker *FakeGoMethodsInvoker) IsValid(methodName string, methodReceiver interface{}) bool { + if methodName == "V001_some_non_existing_method_up" { + return false + } -func (driver *FakeGoMethodsDriver) V001_init_organizations_up() error { - driver.InvokedMethods = append(driver.InvokedMethods, "V001_init_organizations_up") - return nil + return true } -func (driver *FakeGoMethodsDriver) V001_init_organizations_down() error { - driver.InvokedMethods = append(driver.InvokedMethods, "V001_init_organizations_down") - return nil +func (invoker *FakeGoMethodsInvoker) Invoke(methodName string, methodReceiver interface{}) error { + invoker.InvokedMethods = append(invoker.InvokedMethods, methodName) -} - -func (driver *FakeGoMethodsDriver) V001_init_users_up() error { - driver.InvokedMethods = append(driver.InvokedMethods, "V001_init_users_up") - return nil -} - -func (driver *FakeGoMethodsDriver) V001_init_users_down() error { - driver.InvokedMethods = append(driver.InvokedMethods, "V001_init_users_down") - return nil + if methodName == "V001_some_failing_method_up" || methodName == "V001_some_failing_method_down" { + return &MethodInvocationFailedError{ + MethodName: methodName, + Err: SomeError{}, + } + } else { + return nil + } } type SomeError struct{} -func (e SomeError) Error() string { return "Some error happened" } -func (driver *FakeGoMethodsDriver) V001_some_failing_method_up() error { - driver.InvokedMethods = append(driver.InvokedMethods, "V001_some_failing_method_up") - return SomeError{} -} - -func (driver *FakeGoMethodsDriver) V001_some_failing_method_down() error { - driver.InvokedMethods = append(driver.InvokedMethods, "V001_some_failing_method_down") - return SomeError{} -} +func (e SomeError) Error() string { return "Some error happened" } func TestMigrate(t *testing.T) { cases := []struct { - name string - file file.File + name string + file file.File expectedInvokedMethods []string - expectedErrors []error - expectRollback bool + expectedErrors []error + expectRollback bool }{ { name: "up migration invokes up methods", - file: file.File { + file: file.File{ Path: "/foobar", FileName: "001_foobar.up.gm", Version: 1, Name: "foobar", Direction: direction.Up, Content: []byte(` + FakeMethodsReceiver V001_init_organizations_up V001_init_users_up `), }, expectedInvokedMethods: []string{"V001_init_organizations_up", "V001_init_users_up"}, - expectedErrors: []error{}, + expectedErrors: []error{}, }, { name: "down migration invoked down methods", - file: file.File { + file: file.File{ Path: "/foobar", FileName: "001_foobar.down.gm", Version: 1, Name: "foobar", Direction: direction.Down, Content: []byte(` + FakeMethodsReceiver V001_init_users_down V001_init_organizations_down `), }, expectedInvokedMethods: []string{"V001_init_users_down", "V001_init_organizations_down"}, - expectedErrors: []error{}, + expectedErrors: []error{}, }, { name: "up migration: non-existing method causes migration not to execute", - file: file.File { + file: file.File{ Path: "/foobar", FileName: "001_foobar.up.gm", Version: 1, Name: "foobar", Direction: direction.Up, Content: []byte(` + FakeMethodsReceiver V001_init_organizations_up V001_init_users_up V001_some_non_existing_method_up `), }, expectedInvokedMethods: []string{}, - expectedErrors: []error{ MissingMethodError("V001_some_non_existing_method_up") }, + expectedErrors: []error{MissingMethodError("V001_some_non_existing_method_up")}, }, { name: "up migration: failing method stops execution", - file: file.File { + file: file.File{ Path: "/foobar", FileName: "001_foobar.up.gm", Version: 1, Name: "foobar", Direction: direction.Up, Content: []byte(` + FakeMethodsReceiver V001_init_organizations_up V001_some_failing_method_up V001_init_users_up @@ -146,20 +118,21 @@ func TestMigrate(t *testing.T) { "V001_init_organizations_up", "V001_some_failing_method_up", }, - expectedErrors: []error{ &MethodInvocationFailedError{ + expectedErrors: []error{&MethodInvocationFailedError{ MethodName: "V001_some_failing_method_up", - Err: SomeError{}, + Err: SomeError{}, }}, }, { name: "down migration: failing method stops migration", - file: file.File { + file: file.File{ Path: "/foobar", FileName: "001_foobar.down.gm", Version: 1, Name: "foobar", Direction: direction.Down, Content: []byte(` + FakeMethodsReceiver V001_init_users_down V001_some_failing_method_down V001_init_organizations_down @@ -169,21 +142,22 @@ func TestMigrate(t *testing.T) { "V001_init_users_down", "V001_some_failing_method_down", }, - expectedErrors: []error{ &MethodInvocationFailedError{ + expectedErrors: []error{&MethodInvocationFailedError{ MethodName: "V001_some_failing_method_down", - Err: SomeError{}, + Err: SomeError{}, }}, }, { - name: "up migration: failing method causes rollback in rollback mode", + name: "up migration: failing method causes rollback in rollback mode", expectRollback: true, - file: file.File { + file: file.File{ Path: "/foobar", FileName: "001_foobar.up.gm", Version: 1, Name: "foobar", Direction: direction.Up, Content: []byte(` + FakeMethodsReceiver V001_init_organizations_up V001_init_users_up V001_some_failing_method_up @@ -196,21 +170,22 @@ func TestMigrate(t *testing.T) { "V001_init_users_down", "V001_init_organizations_down", }, - expectedErrors: []error{ &MethodInvocationFailedError{ + expectedErrors: []error{&MethodInvocationFailedError{ MethodName: "V001_some_failing_method_up", - Err: SomeError{}, + Err: SomeError{}, }}, }, { - name: "down migration: failing method causes rollback in rollback mode", + name: "down migration: failing method causes rollback in rollback mode", expectRollback: true, - file: file.File { + file: file.File{ Path: "/foobar", FileName: "001_foobar.down.gm", Version: 1, Name: "foobar", Direction: direction.Down, Content: []byte(` + FakeMethodsReceiver V001_init_users_down V001_some_failing_method_down V001_init_organizations_down @@ -221,18 +196,20 @@ func TestMigrate(t *testing.T) { "V001_some_failing_method_down", "V001_init_users_up", }, - expectedErrors: []error{ &MethodInvocationFailedError{ + expectedErrors: []error{&MethodInvocationFailedError{ MethodName: "V001_some_failing_method_down", - Err: SomeError{}, + Err: SomeError{}, }}, }, - } + RegisterMethodsReceiver("FakeMethodsReceiver", "") + for _, c := range cases { migrator := Migrator{} - d := &FakeGoMethodsDriver{Migrator: migrator, InvokedMethods:[]string{}} - migrator.MigrationMethodsReceiver = d + fakeInvoker := &FakeGoMethodsInvoker{InvokedMethods: []string{}} + + migrator.MethodInvoker = fakeInvoker migrator.RollbackOnFailure = c.expectRollback pipe := pipep.New() @@ -243,9 +220,9 @@ func TestMigrate(t *testing.T) { errs := pipep.ReadErrors(pipe) var failed bool - if !reflect.DeepEqual(d.InvokedMethods, c.expectedInvokedMethods) { + if !reflect.DeepEqual(fakeInvoker.InvokedMethods, c.expectedInvokedMethods) { failed = true - t.Errorf("case '%s': FAILED\nexpected invoked methods %v\nbut got %v", c.name, c.expectedInvokedMethods, d.InvokedMethods) + t.Errorf("case '%s': FAILED\nexpected invoked methods %v\nbut got %v", c.name, c.expectedInvokedMethods, fakeInvoker.InvokedMethods) } if !reflect.DeepEqual(errs, c.expectedErrors) { failed = true @@ -258,11 +235,9 @@ func TestMigrate(t *testing.T) { } } - - func TestGetRollbackToMethod(t *testing.T) { cases := []struct { - method string + method string expectedRollbackMethod string }{ {"some_method_up", "some_method_down"}, @@ -282,7 +257,7 @@ func TestGetRollbackToMethod(t *testing.T) { } func TestReverseInPlace(t *testing.T) { - methods := []string { + methods := []string{ "method1_down", "method2_down", "method3_down", @@ -290,7 +265,7 @@ func TestReverseInPlace(t *testing.T) { "method5_down", } - expectedReversedMethods := []string { + expectedReversedMethods := []string{ "method5_down", "method4_down", "method3_down", @@ -304,4 +279,3 @@ func TestReverseInPlace(t *testing.T) { t.Errorf("Expected reverse methods %v but got %v", expectedReversedMethods, methods) } } - diff --git a/driver/gomethods/gomethods_registry.go b/driver/gomethods/gomethods_registry.go new file mode 100644 index 00000000..78ac84c4 --- /dev/null +++ b/driver/gomethods/gomethods_registry.go @@ -0,0 +1,30 @@ +package gomethods + +import ( + "sync" +) + +var methodsReceiversMu sync.Mutex +var methodsReceivers = make(map[string]interface{}) + +// Registers a methods receiver object so it can be created from its name. Users of gomethods migration drivers should +// call this method to register objects with their migration methods before executing the migration +func RegisterMethodsReceiver(name string, receiver interface{}) { + methodsReceiversMu.Lock() + defer methodsReceiversMu.Unlock() + if receiver == nil { + panic("Go methods: Register receiver object is nil") + } + if _, dup := methodsReceivers[name]; dup { + panic("Go methods: Register called twice for receiver " + name) + } + methodsReceivers[name] = receiver +} + +// Retrieves a registered driver by name +func GetMethodsReceiver(name string) interface{} { + methodsReceiversMu.Lock() + defer methodsReceiversMu.Unlock() + receiver := methodsReceivers[name] + return receiver +} diff --git a/driver/gomethods/mongodb/mongodb.go b/driver/gomethods/mongodb/mongodb.go new file mode 100644 index 00000000..0acad176 --- /dev/null +++ b/driver/gomethods/mongodb/mongodb.go @@ -0,0 +1,134 @@ +package mongodb + +import ( + "errors" + "github.com/dimag-jfrog/migrate/driver" + "github.com/dimag-jfrog/migrate/driver/gomethods" + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/migrate/direction" + "gopkg.in/mgo.v2" + "gopkg.in/mgo.v2/bson" + "reflect" + "strings" +) + +const MIGRATE_DB = "db_migrations" +const MIGRATE_C = "db_migrations" + +type MongoDbGoMethodsDriver struct { + Session *mgo.Session + DbName string + + migrator gomethods.Migrator +} + +func init() { + driver.RegisterDriver("mongodb", &MongoDbGoMethodsDriver{}) +} + +type DbMigration struct { + Id bson.ObjectId `bson:"_id,omitempty"` + Version uint64 `bson:"version"` +} + +func (driver *MongoDbGoMethodsDriver) Initialize(url string) error { + urlWithoutScheme := strings.SplitN(url, "mongodb://", 2) + if len(urlWithoutScheme) != 2 { + return errors.New("invalid mongodb:// scheme") + } + + session, err := mgo.Dial(url) + if err != nil { + return err + } + session.SetMode(mgo.Monotonic, true) + + driver.Session = session + driver.DbName = MIGRATE_DB + driver.migrator = gomethods.Migrator{MethodInvoker: driver} + + return nil +} + +func (driver *MongoDbGoMethodsDriver) Close() error { + if driver.Session != nil { + driver.Session.Close() + } + return nil +} + +func (driver *MongoDbGoMethodsDriver) FilenameExtension() string { + return "mgo" +} + +func (driver *MongoDbGoMethodsDriver) Version() (uint64, error) { + var latestMigration DbMigration + c := driver.Session.DB(driver.DbName).C(MIGRATE_C) + + err := c.Find(bson.M{}).Sort("-version").One(&latestMigration) + + switch { + case err == mgo.ErrNotFound: + return 0, nil + case err != nil: + return 0, err + default: + return latestMigration.Version, nil + } +} +func (driver *MongoDbGoMethodsDriver) Migrate(f file.File, pipe chan interface{}) { + defer close(pipe) + pipe <- f + + err := driver.migrator.Migrate(f, pipe) + if err != nil { + return + } + + migrate_c := driver.Session.DB(driver.DbName).C(MIGRATE_C) + + if f.Direction == direction.Up { + id := bson.NewObjectId() + dbMigration := DbMigration{Id: id, Version: f.Version} + + err := migrate_c.Insert(dbMigration) + if err != nil { + pipe <- err + return + } + + } else if f.Direction == direction.Down { + err := migrate_c.Remove(bson.M{"version": f.Version}) + if err != nil { + pipe <- err + return + } + } +} + +func (driver *MongoDbGoMethodsDriver) IsValid(methodName string, methodsReceiver interface{}) bool { + return reflect.ValueOf(methodsReceiver).MethodByName(methodName).IsValid() +} + +func (driver *MongoDbGoMethodsDriver) Invoke(methodName string, methodsReceiver interface{}) error { + name := methodName + migrateMethod := reflect.ValueOf(methodsReceiver).MethodByName(name) + if !migrateMethod.IsValid() { + return gomethods.MissingMethodError(methodName) + } + + retValues := migrateMethod.Call([]reflect.Value{reflect.ValueOf(driver.Session)}) + if len(retValues) != 1 { + return gomethods.WrongMethodSignatureError(name) + } + + if !retValues[0].IsNil() { + err, ok := retValues[0].Interface().(error) + if !ok { + return gomethods.WrongMethodSignatureError(name) + } + return &gomethods.MethodInvocationFailedError{MethodName: name, Err: err} + } + + return nil +} diff --git a/driver/gomethods/mongodb/mongodb_template.go b/driver/gomethods/mongodb/mongodb_template.go deleted file mode 100644 index 1edbe78e..00000000 --- a/driver/gomethods/mongodb/mongodb_template.go +++ /dev/null @@ -1,107 +0,0 @@ -package mongodb - -import ( - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" - "errors" - "strings" - "github.com/dimag-jfrog/migrate/migrate/direction" - "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/driver/gomethods" -) - -const MIGRATE_C = "db_migrations" - - -// This is not a real driver since the Initialize method requires a gomethods.Migrator -// The real driver will contain the DriverTemplate and implement all the custom migration Golang methods -// See example in usage_examples for details -type DriverTemplate struct { - Session *mgo.Session - DbName string - - migrator gomethods.Migrator -} - - -type DbMigration struct { - Id bson.ObjectId `bson:"_id,omitempty"` - Version uint64 `bson:"version"` -} - -func (driver *DriverTemplate) Initialize(url, dbName string, migrator gomethods.Migrator) error { - urlWithoutScheme := strings.SplitN(url, "mongodb://", 2) - if len(urlWithoutScheme) != 2 { - return errors.New("invalid mongodb:// scheme") - } - - session, err := mgo.Dial(url) - if err != nil { - return err - } - session.SetMode(mgo.Monotonic, true) - - driver.Session = session - driver.DbName = dbName - driver.migrator = migrator - - return nil -} - -func (driver *DriverTemplate) Close() error { - if driver.Session != nil { - driver.Session.Close() - } - return nil -} - -func (driver *DriverTemplate) FilenameExtension() string { - return "mgo" -} - - -func (driver *DriverTemplate) Version() (uint64, error) { - var latestMigration DbMigration - c := driver.Session.DB(driver.DbName).C(MIGRATE_C) - - - err := c.Find(bson.M{}).Sort("-version").One(&latestMigration) - - switch { - case err == mgo.ErrNotFound: - return 0, nil - case err != nil: - return 0, err - default: - return latestMigration.Version, nil - } -} -func (driver *DriverTemplate) Migrate(f file.File, pipe chan interface{}) { - defer close(pipe) - pipe <- f - - err := driver.migrator.Migrate(f, pipe) - if err != nil { - return - } - - migrate_c := driver.Session.DB(driver.DbName).C(MIGRATE_C) - - if f.Direction == direction.Up { - id := bson.NewObjectId() - dbMigration := DbMigration{Id: id, Version: f.Version} - - err := migrate_c.Insert(dbMigration) - if err != nil { - pipe <- err - return - } - - } else if f.Direction == direction.Down { - err := migrate_c.Remove(bson.M{"version": f.Version}) - if err != nil { - pipe <- err - return - } - } -} diff --git a/driver/gomethods/mongodb_example/methods_receiver.go b/driver/gomethods/mongodb_example/methods_receiver.go new file mode 100644 index 00000000..24433e96 --- /dev/null +++ b/driver/gomethods/mongodb_example/methods_receiver.go @@ -0,0 +1,129 @@ +package mongodb_example + +import ( + "github.com/dimag-jfrog/migrate/driver/gomethods" + "gopkg.in/mgo.v2" + "gopkg.in/mgo.v2/bson" + "time" +) + +type MyMgoMethodsReceiver struct { +} + +func init() { + gomethods.RegisterMethodsReceiver("MyMgoMethodsReceiver", &MyMgoMethodsReceiver{}) +} + +// Here goes the specific mongodb golang methods driver logic + +const DB_NAME = "test" +const SHORT_DATE_LAYOUT = "2000-Jan-01" +const USERS_C = "users" +const ORGANIZATIONS_C = "organizations" + +type Organization struct { + Id bson.ObjectId `bson:"_id,omitempty"` + Name string `bson:"name"` + Location string `bson:"location"` + DateFounded time.Time `bson:"date_founded"` +} + +type Organization_v2 struct { + Id bson.ObjectId `bson:"_id,omitempty"` + Name string `bson:"name"` + Headquarters string `bson:"headquarters"` + DateFounded time.Time `bson:"date_founded"` +} + +type User struct { + Id bson.ObjectId `bson:"_id"` + Name string `bson:"name"` +} + +var OrganizationIds []bson.ObjectId = []bson.ObjectId{ + bson.NewObjectId(), + bson.NewObjectId(), + bson.NewObjectId(), +} + +var UserIds []bson.ObjectId = []bson.ObjectId{ + bson.NewObjectId(), + bson.NewObjectId(), + bson.NewObjectId(), +} + +func (r *MyMgoMethodsReceiver) V001_init_organizations_up(session *mgo.Session) error { + date1, _ := time.Parse(SHORT_DATE_LAYOUT, "1994-Jul-05") + date2, _ := time.Parse(SHORT_DATE_LAYOUT, "1998-Sep-04") + date3, _ := time.Parse(SHORT_DATE_LAYOUT, "2008-Apr-28") + + orgs := []Organization{ + {Id: OrganizationIds[0], Name: "Amazon", Location: "Seattle", DateFounded: date1}, + {Id: OrganizationIds[1], Name: "Google", Location: "Mountain View", DateFounded: date2}, + {Id: OrganizationIds[2], Name: "JFrog", Location: "Santa Clara", DateFounded: date3}, + } + + for _, org := range orgs { + err := session.DB(DB_NAME).C(ORGANIZATIONS_C).Insert(org) + if err != nil { + return err + } + } + return nil +} + +func (r *MyMgoMethodsReceiver) V001_init_organizations_down(session *mgo.Session) error { + return session.DB(DB_NAME).C(ORGANIZATIONS_C).DropCollection() +} + +func (r *MyMgoMethodsReceiver) V001_init_users_up(session *mgo.Session) error { + users := []User{ + {Id: UserIds[0], Name: "Alex"}, + {Id: UserIds[1], Name: "Beatrice"}, + {Id: UserIds[2], Name: "Cleo"}, + } + + for _, user := range users { + err := session.DB(DB_NAME).C(USERS_C).Insert(user) + if err != nil { + return err + } + } + return nil +} + +func (r *MyMgoMethodsReceiver) V001_init_users_down(session *mgo.Session) error { + return session.DB(DB_NAME).C(USERS_C).DropCollection() +} + +func (r *MyMgoMethodsReceiver) V002_organizations_rename_location_field_to_headquarters_up(session *mgo.Session) error { + c := session.DB(DB_NAME).C(ORGANIZATIONS_C) + + _, err := c.UpdateAll(nil, bson.M{"$rename": bson.M{"location": "headquarters"}}) + return err +} + +func (r *MyMgoMethodsReceiver) V002_organizations_rename_location_field_to_headquarters_down(session *mgo.Session) error { + c := session.DB(DB_NAME).C(ORGANIZATIONS_C) + + _, err := c.UpdateAll(nil, bson.M{"$rename": bson.M{"headquarters": "location"}}) + return err +} + +func (r *MyMgoMethodsReceiver) V002_change_user_cleo_to_cleopatra_up(session *mgo.Session) error { + c := session.DB(DB_NAME).C(USERS_C) + + colQuerier := bson.M{"name": "Cleo"} + change := bson.M{"$set": bson.M{"name": "Cleopatra"}} + + return c.Update(colQuerier, change) +} + +func (r *MyMgoMethodsReceiver) V002_change_user_cleo_to_cleopatra_down(session *mgo.Session) error { + c := session.DB(DB_NAME).C(USERS_C) + + colQuerier := bson.M{"name": "Cleopatra"} + change := bson.M{"$set": bson.M{"name": "Cleo"}} + + return c.Update(colQuerier, change) +} diff --git a/driver/gomethods/usage_examples/mongodb_test.go b/driver/gomethods/mongodb_example/mongodb_test.go similarity index 71% rename from driver/gomethods/usage_examples/mongodb_test.go rename to driver/gomethods/mongodb_example/mongodb_test.go index a57c51da..2ef2797d 100644 --- a/driver/gomethods/usage_examples/mongodb_test.go +++ b/driver/gomethods/mongodb_example/mongodb_test.go @@ -1,4 +1,4 @@ -package usage_examples +package mongodb_example import ( "testing" @@ -6,24 +6,24 @@ import ( "github.com/dimag-jfrog/migrate/file" "github.com/dimag-jfrog/migrate/migrate/direction" + "github.com/dimag-jfrog/migrate/driver/gomethods" + "github.com/dimag-jfrog/migrate/driver/gomethods/mongodb" pipep "github.com/dimag-jfrog/migrate/pipe" "reflect" "time" - "github.com/dimag-jfrog/migrate/driver/gomethods" ) - type ExpectedMigrationResult struct { - Organizations []Organization + Organizations []Organization Organizations_v2 []Organization_v2 - Users []User - Errors []error + Users []User + Errors []error } func RunMigrationAndAssertResult( t *testing.T, title string, - d *GoMethodsMongoDbDriver, + d *mongodb.MongoDbGoMethodsDriver, file file.File, expected *ExpectedMigrationResult) { @@ -53,6 +53,10 @@ func RunMigrationAndAssertResult( t.Fatal("Failed to query Users collection") } + if !reflect.DeepEqual(expected.Errors, errs) { + t.Fatalf("Migration '%s': FAILED\nexpected errors %v\nbut got %v", title, expected.Errors, errs) + } + if !reflect.DeepEqual(expected.Organizations, actualOrganizations) { t.Fatalf("Migration '%s': FAILED\nexpected organizations %v\nbut got %v", title, expected.Organizations, actualOrganizations) } @@ -65,13 +69,9 @@ func RunMigrationAndAssertResult( t.Fatalf("Migration '%s': FAILED\nexpected users %v\nbut got %v", title, expected.Users, actualUsers) } - if !reflect.DeepEqual(expected.Errors, errs) { - t.Fatalf("Migration '%s': FAILED\nexpected errors %v\nbut got %v", title, expected.Errors, errs) - } t.Logf("Migration '%s': PASSED", title) } - func TestMigrate(t *testing.T) { //host := os.Getenv("MONGODB_PORT_27017_TCP_ADDR") //port := os.Getenv("MONGODB_PORT_27017_TCP_PORT") @@ -79,7 +79,9 @@ func TestMigrate(t *testing.T) { port := "27017" driverUrl := "mongodb://" + host + ":" + port - d := &GoMethodsMongoDbDriver{} + //gomethods.RegisterMethodsReceiver("MyMgoMethodsReceiver", &MyMgoMethodsReceiver{}) + d := &mongodb.MongoDbGoMethodsDriver{} + if err := d.Initialize(driverUrl); err != nil { t.Fatal(err) } @@ -93,8 +95,8 @@ func TestMigrate(t *testing.T) { date3, _ := time.Parse(SHORT_DATE_LAYOUT, "2008-Apr-28") migrations := []struct { - name string - file file.File + name string + file file.File expectedResult ExpectedMigrationResult }{ { @@ -106,15 +108,16 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Up, Content: []byte(` + MyMgoMethodsReceiver V001_init_organizations_up V001_init_users_up `), }, expectedResult: ExpectedMigrationResult{ Organizations: []Organization{ - {Id: OrganizationIds[0], Name: "Amazon", Location:"Seattle", DateFounded: date1}, - {Id: OrganizationIds[1], Name: "Google", Location:"Mountain View", DateFounded: date2}, - {Id: OrganizationIds[2], Name: "JFrog", Location:"Santa Clara", DateFounded: date3}, + {Id: OrganizationIds[0], Name: "Amazon", Location: "Seattle", DateFounded: date1}, + {Id: OrganizationIds[1], Name: "Google", Location: "Mountain View", DateFounded: date2}, + {Id: OrganizationIds[2], Name: "JFrog", Location: "Santa Clara", DateFounded: date3}, }, Organizations_v2: []Organization_v2{}, Users: []User{ @@ -134,6 +137,7 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Up, Content: []byte(` + MyMgoMethodsReceiver V002_organizations_rename_location_field_to_headquarters_up V002_change_user_cleo_to_cleopatra_up `), @@ -141,9 +145,9 @@ func TestMigrate(t *testing.T) { expectedResult: ExpectedMigrationResult{ Organizations: []Organization{}, Organizations_v2: []Organization_v2{ - {Id: OrganizationIds[0], Name: "Amazon", Headquarters:"Seattle", DateFounded: date1}, - {Id: OrganizationIds[1], Name: "Google", Headquarters:"Mountain View", DateFounded: date2}, - {Id: OrganizationIds[2], Name: "JFrog", Headquarters:"Santa Clara", DateFounded: date3}, + {Id: OrganizationIds[0], Name: "Amazon", Headquarters: "Seattle", DateFounded: date1}, + {Id: OrganizationIds[1], Name: "Google", Headquarters: "Mountain View", DateFounded: date2}, + {Id: OrganizationIds[2], Name: "JFrog", Headquarters: "Santa Clara", DateFounded: date3}, }, Users: []User{ {Id: UserIds[0], Name: "Alex"}, @@ -162,15 +166,16 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Down, Content: []byte(` + MyMgoMethodsReceiver V002_change_user_cleo_to_cleopatra_down V002_organizations_rename_location_field_to_headquarters_down `), }, expectedResult: ExpectedMigrationResult{ Organizations: []Organization{ - {Id: OrganizationIds[0], Name: "Amazon", Location:"Seattle", DateFounded: date1}, - {Id: OrganizationIds[1], Name: "Google", Location:"Mountain View", DateFounded: date2}, - {Id: OrganizationIds[2], Name: "JFrog", Location:"Santa Clara", DateFounded: date3}, + {Id: OrganizationIds[0], Name: "Amazon", Location: "Seattle", DateFounded: date1}, + {Id: OrganizationIds[1], Name: "Google", Location: "Mountain View", DateFounded: date2}, + {Id: OrganizationIds[2], Name: "JFrog", Location: "Santa Clara", DateFounded: date3}, }, Organizations_v2: []Organization_v2{}, Users: []User{ @@ -190,15 +195,16 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Down, Content: []byte(` + MyMgoMethodsReceiver V001_init_users_down V001_init_organizations_down `), }, expectedResult: ExpectedMigrationResult{ - Organizations: []Organization{}, + Organizations: []Organization{}, Organizations_v2: []Organization_v2{}, - Users: []User{}, - Errors: []error{}, + Users: []User{}, + Errors: []error{}, }, }, { @@ -210,16 +216,37 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Up, Content: []byte(` + MyMgoMethodsReceiver V001_init_organizations_up V001_init_users_up v001_non_existing_method_up `), }, expectedResult: ExpectedMigrationResult{ - Organizations: []Organization{}, + Organizations: []Organization{}, + Organizations_v2: []Organization_v2{}, + Users: []User{}, + Errors: []error{gomethods.MissingMethodError("v001_non_existing_method_up")}, + }, + }, + { + name: "v0 -> v1: not defined message receiver", + file: file.File{ + Path: "/foobar", + FileName: "001_foobar.up.gm", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V001_init_organizations_up + V001_init_users_up + `), + }, + expectedResult: ExpectedMigrationResult{ + Organizations: []Organization{}, Organizations_v2: []Organization_v2{}, - Users: []User{}, - Errors: []error{ gomethods.MissingMethodError("v001_non_existing_method_up") }, + Users: []User{}, + Errors: []error{gomethods.UnregisteredMethodsReceiverError("V001_init_organizations_up")}, }, }, } diff --git a/driver/gomethods/usage_examples/mongodb.go b/driver/gomethods/usage_examples/mongodb.go deleted file mode 100644 index e7f30c0a..00000000 --- a/driver/gomethods/usage_examples/mongodb.go +++ /dev/null @@ -1,140 +0,0 @@ -package usage_examples - -import ( - "github.com/dimag-jfrog/migrate/driver" - "github.com/dimag-jfrog/migrate/driver/gomethods" - "github.com/dimag-jfrog/migrate/driver/gomethods/mongodb" - "gopkg.in/mgo.v2/bson" - "time" -) - -// This boilerplate part is necessary and the same -// regardless of the specific mongodb golang methods driver - -type GoMethodsMongoDbDriver struct { - mongodb.DriverTemplate -} - -func (d *GoMethodsMongoDbDriver) Initialize(url string) error { - return d.DriverTemplate.Initialize(url, DB_NAME, gomethods.Migrator{MigrationMethodsReceiver: d}) -} - -func init() { - driver.RegisterDriver("mongodb", &GoMethodsMongoDbDriver{mongodb.DriverTemplate{}}) -} - - - -// Here goes the specific mongodb golang methods driver logic - -const DB_NAME = "test" -const SHORT_DATE_LAYOUT = "2000-Jan-01" -const USERS_C = "users" -const ORGANIZATIONS_C = "organizations" - -type Organization struct { - Id bson.ObjectId `bson:"_id,omitempty"` - Name string `bson:"name"` - Location string `bson:"location"` - DateFounded time.Time `bson:"date_founded"` -} - -type Organization_v2 struct { - Id bson.ObjectId `bson:"_id,omitempty"` - Name string `bson:"name"` - Headquarters string `bson:"headquarters"` - DateFounded time.Time `bson:"date_founded"` -} - -type User struct { - Id bson.ObjectId `bson:"_id"` - Name string `bson:"name"` -} - -var OrganizationIds []bson.ObjectId = []bson.ObjectId{ - bson.NewObjectId(), - bson.NewObjectId(), - bson.NewObjectId(), -} - -var UserIds []bson.ObjectId = []bson.ObjectId{ - bson.NewObjectId(), - bson.NewObjectId(), - bson.NewObjectId(), -} - -func (m *GoMethodsMongoDbDriver) V001_init_organizations_up() error { - date1, _ := time.Parse(SHORT_DATE_LAYOUT, "1994-Jul-05") - date2, _ := time.Parse(SHORT_DATE_LAYOUT, "1998-Sep-04") - date3, _ := time.Parse(SHORT_DATE_LAYOUT, "2008-Apr-28") - - orgs := []Organization{ - {Id: OrganizationIds[0], Name: "Amazon", Location:"Seattle", DateFounded: date1}, - {Id: OrganizationIds[1], Name: "Google", Location:"Mountain View", DateFounded: date2}, - {Id: OrganizationIds[2], Name: "JFrog", Location:"Santa Clara", DateFounded: date3}, - } - - for _, org := range orgs { - err := m.Session.DB(DB_NAME).C(ORGANIZATIONS_C).Insert(org) - if err != nil { - return err - } - } - return nil -} - -func (m *GoMethodsMongoDbDriver) V001_init_organizations_down() error { - return m.Session.DB(DB_NAME).C(ORGANIZATIONS_C).DropCollection() -} - -func (m *GoMethodsMongoDbDriver) V001_init_users_up() error { - users := []User{ - {Id: UserIds[0], Name: "Alex"}, - {Id: UserIds[1], Name: "Beatrice"}, - {Id: UserIds[2], Name: "Cleo"}, - } - - for _, user := range users { - err := m.Session.DB(DB_NAME).C(USERS_C).Insert(user) - if err != nil { - return err - } - } - return nil -} - -func (m *GoMethodsMongoDbDriver) V001_init_users_down() error { - return m.Session.DB(DB_NAME).C(USERS_C).DropCollection() -} - -func (m *GoMethodsMongoDbDriver) V002_organizations_rename_location_field_to_headquarters_up() error { - c := m.Session.DB(DB_NAME).C(ORGANIZATIONS_C) - - _, err := c.UpdateAll(nil, bson.M{"$rename": bson.M{"location": "headquarters"}}) - return err -} - -func (m *GoMethodsMongoDbDriver) V002_organizations_rename_location_field_to_headquarters_down() error { - c := m.Session.DB(DB_NAME).C(ORGANIZATIONS_C) - - _, err := c.UpdateAll(nil, bson.M{"$rename": bson.M{"headquarters": "location"}}) - return err -} - -func (m *GoMethodsMongoDbDriver) V002_change_user_cleo_to_cleopatra_up() error { - c := m.Session.DB(DB_NAME).C(USERS_C) - - colQuerier := bson.M{"name": "Cleo"} - change := bson.M{"$set": bson.M{"name": "Cleopatra"}} - - return c.Update(colQuerier, change) -} - -func (m *GoMethodsMongoDbDriver) V002_change_user_cleo_to_cleopatra_down() error { - c := m.Session.DB(DB_NAME).C(USERS_C) - - colQuerier := bson.M{"name": "Cleopatra"} - change := bson.M{"$set": bson.M{"name": "Cleo",}} - - return c.Update(colQuerier, change) -} \ No newline at end of file From 866ff82ffe692054b79b92c21a5b31a24c669e23 Mon Sep 17 00:00:00 2001 From: dimag Date: Tue, 9 Aug 2016 21:16:21 +0300 Subject: [PATCH 22/42] - registering method receiver directly for the driver - currently each driver contains only a single method receiver: - enforcing method receiver pre-registration on go methods driver initialization - Method receiver name can be removed from files format - passing the DbName parameter inside the method receiver for the go methods driver --- driver/gomethods/gomethods_migrator.go | 53 ++++++------------ driver/gomethods/gomethods_migrator_test.go | 13 +---- driver/gomethods/gomethods_registry.go | 37 ++++++++----- driver/gomethods/mongodb/mongodb.go | 55 +++++++++++++++---- .../mongodb_example/methods_receiver.go | 20 +++++-- .../gomethods/mongodb_example/mongodb_test.go | 40 +++++--------- 6 files changed, 116 insertions(+), 102 deletions(-) diff --git a/driver/gomethods/gomethods_migrator.go b/driver/gomethods/gomethods_migrator.go index 2ee01925..70d57c38 100644 --- a/driver/gomethods/gomethods_migrator.go +++ b/driver/gomethods/gomethods_migrator.go @@ -1,22 +1,15 @@ package gomethods import ( - //"bytes" "bufio" - "database/sql/driver" "fmt" + "github.com/dimag-jfrog/migrate/driver" "github.com/dimag-jfrog/migrate/file" "os" "path" "strings" ) -type UnregisteredMethodsReceiverError string - -func (e UnregisteredMethodsReceiverError) Error() string { - return "Unregistered methods receiver: " + string(e) -} - type MissingMethodError string func (e MissingMethodError) Error() string { return "Non existing migrate method: " + string(e) } @@ -37,13 +30,16 @@ func (e *MethodInvocationFailedError) Error() string { } type MigrationMethodInvoker interface { - IsValid(methodName string, methodReceiver interface{}) bool - Invoke(methodName string, methodReceiver interface{}) error + IsValid(methodName string) bool + Invoke(methodName string) error } type GoMethodsDriver interface { driver.Driver + MigrationMethodInvoker + MethodsReceiver() interface{} + SetMethodsReceiver(r interface{}) error } type Migrator struct { @@ -52,7 +48,7 @@ type Migrator struct { } func (m *Migrator) Migrate(f file.File, pipe chan interface{}) error { - methods, methodsReceiver, err := m.getMigrationMethods(f) + methods, err := m.getMigrationMethods(f) if err != nil { pipe <- err return err @@ -60,7 +56,7 @@ func (m *Migrator) Migrate(f file.File, pipe chan interface{}) error { for i, methodName := range methods { pipe <- methodName - err := m.MethodInvoker.Invoke(methodName, methodsReceiver) + err := m.MethodInvoker.Invoke(methodName) if err != nil { pipe <- err if !m.RollbackOnFailure { @@ -71,12 +67,12 @@ func (m *Migrator) Migrate(f file.File, pipe chan interface{}) error { for j := i - 1; j >= 0; j-- { rollbackToMethodName := getRollbackToMethod(methods[j]) if rollbackToMethodName == "" || - !m.MethodInvoker.IsValid(rollbackToMethodName, methodsReceiver) { + !m.MethodInvoker.IsValid(rollbackToMethodName) { continue } pipe <- rollbackToMethodName - err = m.MethodInvoker.Invoke(rollbackToMethodName, methodsReceiver) + err = m.MethodInvoker.Invoke(rollbackToMethodName) if err != nil { pipe <- err break @@ -122,20 +118,17 @@ func getFileLines(file file.File) ([]string, error) { } return lines, nil } else { - //n := bytes.IndexByte(file.Content, 0) - //n := bytes.Index(file.Content, []byte{0}) - //s := string(file.Content[:n]) s := string(file.Content) return strings.Split(s, "\n"), nil } } -func (m *Migrator) getMigrationMethods(f file.File) (methods []string, methodsReceiver interface{}, err error) { +func (m *Migrator) getMigrationMethods(f file.File) (methods []string, err error) { var lines []string lines, err = getFileLines(f) if err != nil { - return nil, nil, err + return nil, err } for _, line := range lines { @@ -146,24 +139,14 @@ func (m *Migrator) getMigrationMethods(f file.File) (methods []string, methodsRe continue } - if methodsReceiver == nil { - receiverName := line - methodsReceiver = GetMethodsReceiver(receiverName) - if methodsReceiver == nil { - return nil, nil, UnregisteredMethodsReceiverError(receiverName) - } - continue - - } else { - methodName := line - if !m.MethodInvoker.IsValid(methodName, methodsReceiver) { - return nil, nil, MissingMethodError(methodName) - } - - methods = append(methods, methodName) + methodName := line + if !m.MethodInvoker.IsValid(methodName) { + return nil, MissingMethodError(methodName) } + + methods = append(methods, methodName) } - return methods, methodsReceiver, nil + return methods, nil } diff --git a/driver/gomethods/gomethods_migrator_test.go b/driver/gomethods/gomethods_migrator_test.go index eb33103f..db2a8173 100644 --- a/driver/gomethods/gomethods_migrator_test.go +++ b/driver/gomethods/gomethods_migrator_test.go @@ -14,7 +14,7 @@ type FakeGoMethodsInvoker struct { InvokedMethods []string } -func (invoker *FakeGoMethodsInvoker) IsValid(methodName string, methodReceiver interface{}) bool { +func (invoker *FakeGoMethodsInvoker) IsValid(methodName string) bool { if methodName == "V001_some_non_existing_method_up" { return false } @@ -22,7 +22,7 @@ func (invoker *FakeGoMethodsInvoker) IsValid(methodName string, methodReceiver i return true } -func (invoker *FakeGoMethodsInvoker) Invoke(methodName string, methodReceiver interface{}) error { +func (invoker *FakeGoMethodsInvoker) Invoke(methodName string) error { invoker.InvokedMethods = append(invoker.InvokedMethods, methodName) if methodName == "V001_some_failing_method_up" || methodName == "V001_some_failing_method_down" { @@ -56,7 +56,6 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Up, Content: []byte(` - FakeMethodsReceiver V001_init_organizations_up V001_init_users_up `), @@ -73,7 +72,6 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Down, Content: []byte(` - FakeMethodsReceiver V001_init_users_down V001_init_organizations_down `), @@ -90,7 +88,6 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Up, Content: []byte(` - FakeMethodsReceiver V001_init_organizations_up V001_init_users_up V001_some_non_existing_method_up @@ -108,7 +105,6 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Up, Content: []byte(` - FakeMethodsReceiver V001_init_organizations_up V001_some_failing_method_up V001_init_users_up @@ -132,7 +128,6 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Down, Content: []byte(` - FakeMethodsReceiver V001_init_users_down V001_some_failing_method_down V001_init_organizations_down @@ -157,7 +152,6 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Up, Content: []byte(` - FakeMethodsReceiver V001_init_organizations_up V001_init_users_up V001_some_failing_method_up @@ -185,7 +179,6 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Down, Content: []byte(` - FakeMethodsReceiver V001_init_users_down V001_some_failing_method_down V001_init_organizations_down @@ -203,8 +196,6 @@ func TestMigrate(t *testing.T) { }, } - RegisterMethodsReceiver("FakeMethodsReceiver", "") - for _, c := range cases { migrator := Migrator{} fakeInvoker := &FakeGoMethodsInvoker{InvokedMethods: []string{}} diff --git a/driver/gomethods/gomethods_registry.go b/driver/gomethods/gomethods_registry.go index 78ac84c4..5ceb1287 100644 --- a/driver/gomethods/gomethods_registry.go +++ b/driver/gomethods/gomethods_registry.go @@ -1,30 +1,39 @@ package gomethods import ( + "fmt" + "github.com/dimag-jfrog/migrate/driver" "sync" ) var methodsReceiversMu sync.Mutex -var methodsReceivers = make(map[string]interface{}) -// Registers a methods receiver object so it can be created from its name. Users of gomethods migration drivers should -// call this method to register objects with their migration methods before executing the migration -func RegisterMethodsReceiver(name string, receiver interface{}) { +// Registers a methods receiver for go methods driver +// Users of gomethods migration drivers should call this method +// to register objects with their migration methods before executing the migration +func RegisterMethodsReceiverForDriver(driverName string, receiver interface{}) { methodsReceiversMu.Lock() defer methodsReceiversMu.Unlock() if receiver == nil { panic("Go methods: Register receiver object is nil") } - if _, dup := methodsReceivers[name]; dup { - panic("Go methods: Register called twice for receiver " + name) + + driver := driver.GetDriver(driverName) + if driver == nil { + panic("Go methods: Trying to register receiver for not registered driver " + driverName) } - methodsReceivers[name] = receiver -} -// Retrieves a registered driver by name -func GetMethodsReceiver(name string) interface{} { - methodsReceiversMu.Lock() - defer methodsReceiversMu.Unlock() - receiver := methodsReceivers[name] - return receiver + methodsDriver, ok := driver.(GoMethodsDriver) + if !ok { + panic("Go methods: Trying to register receiver for non go methods driver " + driverName) + } + + if methodsDriver.MethodsReceiver() != nil { + panic("Go methods: Methods receiver already registered for driver " + driverName) + } + + if err := methodsDriver.SetMethodsReceiver(receiver); err != nil { + panic(fmt.Sprintf("Go methods: Failed to set methods receiver for driver %s\nError: %v", + driverName, err)) + } } diff --git a/driver/gomethods/mongodb/mongodb.go b/driver/gomethods/mongodb/mongodb.go index 0acad176..acbc2564 100644 --- a/driver/gomethods/mongodb/mongodb.go +++ b/driver/gomethods/mongodb/mongodb.go @@ -12,14 +12,46 @@ import ( "strings" ) -const MIGRATE_DB = "db_migrations" +type UnregisteredMethodsReceiverError string + +func (e UnregisteredMethodsReceiverError) Error() string { + return "Unregistered methods receiver for driver: " + string(e) +} + +type WrongMethodsReceiverTypeError string + +func (e WrongMethodsReceiverTypeError) Error() string { + return "Wrong methods receiver type for driver: " + string(e) +} + const MIGRATE_C = "db_migrations" +const DRIVER_NAME = "gomethods.mongodb" type MongoDbGoMethodsDriver struct { Session *mgo.Session - DbName string - migrator gomethods.Migrator + methodsReceiver MethodsReceiver + migrator gomethods.Migrator +} + +var _ gomethods.GoMethodsDriver = (*MongoDbGoMethodsDriver)(nil) + +type MethodsReceiver interface { + DbName() string +} + +func (d *MongoDbGoMethodsDriver) MethodsReceiver() interface{} { + return d.methodsReceiver +} + +func (d *MongoDbGoMethodsDriver) SetMethodsReceiver(r interface{}) error { + r1, ok := r.(MethodsReceiver) + if !ok { + return WrongMethodsReceiverTypeError(DRIVER_NAME) + } + + d.methodsReceiver = r1 + return nil } func init() { @@ -32,6 +64,10 @@ type DbMigration struct { } func (driver *MongoDbGoMethodsDriver) Initialize(url string) error { + if driver.methodsReceiver == nil { + return UnregisteredMethodsReceiverError(DRIVER_NAME) + } + urlWithoutScheme := strings.SplitN(url, "mongodb://", 2) if len(urlWithoutScheme) != 2 { return errors.New("invalid mongodb:// scheme") @@ -44,7 +80,6 @@ func (driver *MongoDbGoMethodsDriver) Initialize(url string) error { session.SetMode(mgo.Monotonic, true) driver.Session = session - driver.DbName = MIGRATE_DB driver.migrator = gomethods.Migrator{MethodInvoker: driver} return nil @@ -63,7 +98,7 @@ func (driver *MongoDbGoMethodsDriver) FilenameExtension() string { func (driver *MongoDbGoMethodsDriver) Version() (uint64, error) { var latestMigration DbMigration - c := driver.Session.DB(driver.DbName).C(MIGRATE_C) + c := driver.Session.DB(driver.methodsReceiver.DbName()).C(MIGRATE_C) err := c.Find(bson.M{}).Sort("-version").One(&latestMigration) @@ -85,7 +120,7 @@ func (driver *MongoDbGoMethodsDriver) Migrate(f file.File, pipe chan interface{} return } - migrate_c := driver.Session.DB(driver.DbName).C(MIGRATE_C) + migrate_c := driver.Session.DB(driver.methodsReceiver.DbName()).C(MIGRATE_C) if f.Direction == direction.Up { id := bson.NewObjectId() @@ -106,13 +141,13 @@ func (driver *MongoDbGoMethodsDriver) Migrate(f file.File, pipe chan interface{} } } -func (driver *MongoDbGoMethodsDriver) IsValid(methodName string, methodsReceiver interface{}) bool { - return reflect.ValueOf(methodsReceiver).MethodByName(methodName).IsValid() +func (driver *MongoDbGoMethodsDriver) IsValid(methodName string) bool { + return reflect.ValueOf(driver.methodsReceiver).MethodByName(methodName).IsValid() } -func (driver *MongoDbGoMethodsDriver) Invoke(methodName string, methodsReceiver interface{}) error { +func (driver *MongoDbGoMethodsDriver) Invoke(methodName string) error { name := methodName - migrateMethod := reflect.ValueOf(methodsReceiver).MethodByName(name) + migrateMethod := reflect.ValueOf(driver.methodsReceiver).MethodByName(name) if !migrateMethod.IsValid() { return gomethods.MissingMethodError(methodName) } diff --git a/driver/gomethods/mongodb_example/methods_receiver.go b/driver/gomethods/mongodb_example/methods_receiver.go index 24433e96..6749ae78 100644 --- a/driver/gomethods/mongodb_example/methods_receiver.go +++ b/driver/gomethods/mongodb_example/methods_receiver.go @@ -2,6 +2,8 @@ package mongodb_example import ( "github.com/dimag-jfrog/migrate/driver/gomethods" + _ "github.com/dimag-jfrog/migrate/driver/gomethods" + "github.com/dimag-jfrog/migrate/driver/gomethods/mongodb" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" "time" @@ -10,16 +12,24 @@ import ( type MyMgoMethodsReceiver struct { } +func (r *MyMgoMethodsReceiver) DbName() string { + return DB_NAME +} + +var _ mongodb.MethodsReceiver = (*MyMgoMethodsReceiver)(nil) + func init() { - gomethods.RegisterMethodsReceiver("MyMgoMethodsReceiver", &MyMgoMethodsReceiver{}) + gomethods.RegisterMethodsReceiverForDriver("mongodb", &MyMgoMethodsReceiver{}) } // Here goes the specific mongodb golang methods driver logic -const DB_NAME = "test" -const SHORT_DATE_LAYOUT = "2000-Jan-01" -const USERS_C = "users" -const ORGANIZATIONS_C = "organizations" +const ( + DB_NAME = "test" + SHORT_DATE_LAYOUT = "2000-Jan-01" + USERS_C = "users" + ORGANIZATIONS_C = "organizations" +) type Organization struct { Id bson.ObjectId `bson:"_id,omitempty"` diff --git a/driver/gomethods/mongodb_example/mongodb_test.go b/driver/gomethods/mongodb_example/mongodb_test.go index 2ef2797d..fc21c9c6 100644 --- a/driver/gomethods/mongodb_example/mongodb_test.go +++ b/driver/gomethods/mongodb_example/mongodb_test.go @@ -6,6 +6,7 @@ import ( "github.com/dimag-jfrog/migrate/file" "github.com/dimag-jfrog/migrate/migrate/direction" + "github.com/dimag-jfrog/migrate/driver" "github.com/dimag-jfrog/migrate/driver/gomethods" "github.com/dimag-jfrog/migrate/driver/gomethods/mongodb" pipep "github.com/dimag-jfrog/migrate/pipe" @@ -73,20 +74,30 @@ func RunMigrationAndAssertResult( } func TestMigrate(t *testing.T) { + defer func() { + if r := recover(); r != nil { + t.Fatalf("Test failed on panic: %v", r) + } + }() + //host := os.Getenv("MONGODB_PORT_27017_TCP_ADDR") //port := os.Getenv("MONGODB_PORT_27017_TCP_PORT") host := "127.0.0.1" port := "27017" driverUrl := "mongodb://" + host + ":" + port - //gomethods.RegisterMethodsReceiver("MyMgoMethodsReceiver", &MyMgoMethodsReceiver{}) - d := &mongodb.MongoDbGoMethodsDriver{} + d0 := driver.GetDriver("mongodb") + d, ok := d0.(*mongodb.MongoDbGoMethodsDriver) + if !ok { + t.Fatal("MongoDbGoMethodsDriver has not registered") + } if err := d.Initialize(driverUrl); err != nil { t.Fatal(err) } // Reset DB + d.Session.DB(DB_NAME).C(mongodb.MIGRATE_C).DropCollection() d.Session.DB(DB_NAME).C(ORGANIZATIONS_C).DropCollection() d.Session.DB(DB_NAME).C(USERS_C).DropCollection() @@ -108,7 +119,6 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Up, Content: []byte(` - MyMgoMethodsReceiver V001_init_organizations_up V001_init_users_up `), @@ -137,7 +147,6 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Up, Content: []byte(` - MyMgoMethodsReceiver V002_organizations_rename_location_field_to_headquarters_up V002_change_user_cleo_to_cleopatra_up `), @@ -166,7 +175,6 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Down, Content: []byte(` - MyMgoMethodsReceiver V002_change_user_cleo_to_cleopatra_down V002_organizations_rename_location_field_to_headquarters_down `), @@ -195,7 +203,6 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Down, Content: []byte(` - MyMgoMethodsReceiver V001_init_users_down V001_init_organizations_down `), @@ -216,7 +223,6 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Up, Content: []byte(` - MyMgoMethodsReceiver V001_init_organizations_up V001_init_users_up v001_non_existing_method_up @@ -229,26 +235,6 @@ func TestMigrate(t *testing.T) { Errors: []error{gomethods.MissingMethodError("v001_non_existing_method_up")}, }, }, - { - name: "v0 -> v1: not defined message receiver", - file: file.File{ - Path: "/foobar", - FileName: "001_foobar.up.gm", - Version: 1, - Name: "foobar", - Direction: direction.Up, - Content: []byte(` - V001_init_organizations_up - V001_init_users_up - `), - }, - expectedResult: ExpectedMigrationResult{ - Organizations: []Organization{}, - Organizations_v2: []Organization_v2{}, - Users: []User{}, - Errors: []error{gomethods.UnregisteredMethodsReceiverError("V001_init_organizations_up")}, - }, - }, } for _, m := range migrations { From 834a096bf8877084689440d6d5b433e0c9b2bac1 Mon Sep 17 00:00:00 2001 From: dimag Date: Wed, 10 Aug 2016 14:13:53 +0300 Subject: [PATCH 23/42] Added explicit migration method signature validation --- driver/gomethods/gomethods_migrator.go | 14 +++--- driver/gomethods/gomethods_migrator_test.go | 9 ++-- driver/gomethods/mongodb/mongodb.go | 15 ++++++- .../mongodb_example/methods_receiver.go | 9 ++++ .../gomethods/mongodb_example/mongodb_test.go | 44 ++++++++++++++++++- 5 files changed, 77 insertions(+), 14 deletions(-) diff --git a/driver/gomethods/gomethods_migrator.go b/driver/gomethods/gomethods_migrator.go index 70d57c38..a6900356 100644 --- a/driver/gomethods/gomethods_migrator.go +++ b/driver/gomethods/gomethods_migrator.go @@ -17,7 +17,7 @@ func (e MissingMethodError) Error() string { return "Non existing migrate method type WrongMethodSignatureError string func (e WrongMethodSignatureError) Error() string { - return fmt.Sprintf("Method %s has wrong signature", e) + return fmt.Sprintf("Method %s has wrong signature", string(e)) } type MethodInvocationFailedError struct { @@ -30,7 +30,7 @@ func (e *MethodInvocationFailedError) Error() string { } type MigrationMethodInvoker interface { - IsValid(methodName string) bool + Validate(methodName string) error Invoke(methodName string) error } @@ -66,8 +66,10 @@ func (m *Migrator) Migrate(f file.File, pipe chan interface{}) error { // on failure, try to rollback methods in this migration for j := i - 1; j >= 0; j-- { rollbackToMethodName := getRollbackToMethod(methods[j]) - if rollbackToMethodName == "" || - !m.MethodInvoker.IsValid(rollbackToMethodName) { + if rollbackToMethodName == "" { + continue + } + if err := m.MethodInvoker.Validate(rollbackToMethodName); err != nil { continue } @@ -140,8 +142,8 @@ func (m *Migrator) getMigrationMethods(f file.File) (methods []string, err error } methodName := line - if !m.MethodInvoker.IsValid(methodName) { - return nil, MissingMethodError(methodName) + if err := m.MethodInvoker.Validate(methodName); err != nil { + return nil, err } methods = append(methods, methodName) diff --git a/driver/gomethods/gomethods_migrator_test.go b/driver/gomethods/gomethods_migrator_test.go index db2a8173..9dafb8a7 100644 --- a/driver/gomethods/gomethods_migrator_test.go +++ b/driver/gomethods/gomethods_migrator_test.go @@ -14,12 +14,12 @@ type FakeGoMethodsInvoker struct { InvokedMethods []string } -func (invoker *FakeGoMethodsInvoker) IsValid(methodName string) bool { +func (invoker *FakeGoMethodsInvoker) Validate(methodName string) error { if methodName == "V001_some_non_existing_method_up" { - return false + return MissingMethodError(methodName) } - return true + return nil } func (invoker *FakeGoMethodsInvoker) Invoke(methodName string) error { @@ -30,9 +30,8 @@ func (invoker *FakeGoMethodsInvoker) Invoke(methodName string) error { MethodName: methodName, Err: SomeError{}, } - } else { - return nil } + return nil } type SomeError struct{} diff --git a/driver/gomethods/mongodb/mongodb.go b/driver/gomethods/mongodb/mongodb.go index acbc2564..f07b38b2 100644 --- a/driver/gomethods/mongodb/mongodb.go +++ b/driver/gomethods/mongodb/mongodb.go @@ -141,8 +141,19 @@ func (driver *MongoDbGoMethodsDriver) Migrate(f file.File, pipe chan interface{} } } -func (driver *MongoDbGoMethodsDriver) IsValid(methodName string) bool { - return reflect.ValueOf(driver.methodsReceiver).MethodByName(methodName).IsValid() +func (driver *MongoDbGoMethodsDriver) Validate(methodName string) error { + method := reflect.ValueOf(driver.methodsReceiver).MethodByName(methodName) + if !method.IsValid() { + return gomethods.MissingMethodError(methodName) + } + + methodTemplate := func(*mgo.Session) error { return nil } + + if method.Type() != reflect.TypeOf(methodTemplate) { + return gomethods.WrongMethodSignatureError(methodName) + } + + return nil } func (driver *MongoDbGoMethodsDriver) Invoke(methodName string) error { diff --git a/driver/gomethods/mongodb_example/methods_receiver.go b/driver/gomethods/mongodb_example/methods_receiver.go index 6749ae78..ae96b715 100644 --- a/driver/gomethods/mongodb_example/methods_receiver.go +++ b/driver/gomethods/mongodb_example/methods_receiver.go @@ -137,3 +137,12 @@ func (r *MyMgoMethodsReceiver) V002_change_user_cleo_to_cleopatra_down(session * return c.Update(colQuerier, change) } + +// Wrong signature methods for testing +func (r *MyMgoMethodsReceiver) V001_method_with_wrong_signature_up(s string) error { + return nil +} + +func (r *MyMgoMethodsReceiver) V001_method_with_wrong_signature_down(session *mgo.Session) (bool, error) { + return true, nil +} diff --git a/driver/gomethods/mongodb_example/mongodb_test.go b/driver/gomethods/mongodb_example/mongodb_test.go index fc21c9c6..18862af6 100644 --- a/driver/gomethods/mongodb_example/mongodb_test.go +++ b/driver/gomethods/mongodb_example/mongodb_test.go @@ -215,7 +215,7 @@ func TestMigrate(t *testing.T) { }, }, { - name: "v0 -> v1: with error", + name: "v0 -> v1: missing method aborts migration", file: file.File{ Path: "/foobar", FileName: "001_foobar.up.gm", @@ -235,6 +235,48 @@ func TestMigrate(t *testing.T) { Errors: []error{gomethods.MissingMethodError("v001_non_existing_method_up")}, }, }, + { + name: "v0 -> v1: wrong signature method aborts migration", + file: file.File{ + Path: "/foobar", + FileName: "001_foobar.up.gm", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V001_init_organizations_up + V001_method_with_wrong_signature_up + V001_init_users_up + `), + }, + expectedResult: ExpectedMigrationResult{ + Organizations: []Organization{}, + Organizations_v2: []Organization_v2{}, + Users: []User{}, + Errors: []error{gomethods.WrongMethodSignatureError("V001_method_with_wrong_signature_up")}, + }, + }, + { + name: "v1 -> v0: wrong signature method aborts migration", + file: file.File{ + Path: "/foobar", + FileName: "001_foobar.down.gm", + Version: 1, + Name: "foobar", + Direction: direction.Down, + Content: []byte(` + V001_init_users_down + V001_method_with_wrong_signature_down + V001_init_organizations_down + `), + }, + expectedResult: ExpectedMigrationResult{ + Organizations: []Organization{}, + Organizations_v2: []Organization_v2{}, + Users: []User{}, + Errors: []error{gomethods.WrongMethodSignatureError("V001_method_with_wrong_signature_down")}, + }, + }, } for _, m := range migrations { From e27b9c2a7b27ddb37b630039af7f9e5978f50ccc Mon Sep 17 00:00:00 2001 From: dimag Date: Wed, 10 Aug 2016 17:15:17 +0300 Subject: [PATCH 24/42] - Refactored project structure to be consistent with original project - Added docker support --- docker-compose.yml | 7 +++- .../example}/mongodb_test.go | 20 +++++----- .../example/sample_mongdb_migrator.go} | 37 ++++++++++--------- .../gomethods/gomethods_migrator.go | 7 ---- .../gomethods/gomethods_migrator_test.go | 26 +------------ .../gomethods/gomethods_registry.go | 0 driver/{gomethods => }/mongodb/mongodb.go | 26 ++++++------- 7 files changed, 48 insertions(+), 75 deletions(-) rename driver/{gomethods/mongodb_example => mongodb/example}/mongodb_test.go (95%) rename driver/{gomethods/mongodb_example/methods_receiver.go => mongodb/example/sample_mongdb_migrator.go} (66%) rename driver/{ => mongodb}/gomethods/gomethods_migrator.go (96%) rename driver/{ => mongodb}/gomethods/gomethods_migrator_test.go (92%) rename driver/{ => mongodb}/gomethods/gomethods_registry.go (100%) rename driver/{gomethods => }/mongodb/mongodb.go (80%) diff --git a/docker-compose.yml b/docker-compose.yml index cac3abcf..49684090 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ go: &go image: golang - working_dir: /go/src/github.com/mattes/migrate + working_dir: /go/src/github.com/dimag-jfrog/migrate volumes: - $GOPATH:/go go-test: @@ -10,6 +10,7 @@ go-test: - postgres - mysql - cassandra + - mongo go-build: <<: *go command: sh -c 'go get -v && go build -ldflags ''-s'' -o migrater' @@ -21,6 +22,8 @@ mysql: image: mysql environment: MYSQL_DATABASE: migratetest - MYSQL_ALLOW_EMPTY_PASSWORD: yes + MYSQL_ALLOW_EMPTY_PASSWORD: 'yes' cassandra: image: cassandra:2.2 +mongo: + image: mongo diff --git a/driver/gomethods/mongodb_example/mongodb_test.go b/driver/mongodb/example/mongodb_test.go similarity index 95% rename from driver/gomethods/mongodb_example/mongodb_test.go rename to driver/mongodb/example/mongodb_test.go index 18862af6..c8013d44 100644 --- a/driver/gomethods/mongodb_example/mongodb_test.go +++ b/driver/mongodb/example/mongodb_test.go @@ -1,4 +1,4 @@ -package mongodb_example +package example import ( "testing" @@ -7,9 +7,10 @@ import ( "github.com/dimag-jfrog/migrate/migrate/direction" "github.com/dimag-jfrog/migrate/driver" - "github.com/dimag-jfrog/migrate/driver/gomethods" - "github.com/dimag-jfrog/migrate/driver/gomethods/mongodb" + "github.com/dimag-jfrog/migrate/driver/mongodb" + "github.com/dimag-jfrog/migrate/driver/mongodb/gomethods" pipep "github.com/dimag-jfrog/migrate/pipe" + "os" "reflect" "time" ) @@ -24,7 +25,7 @@ type ExpectedMigrationResult struct { func RunMigrationAndAssertResult( t *testing.T, title string, - d *mongodb.MongoDbGoMethodsDriver, + d *mongodb.Driver, file file.File, expected *ExpectedMigrationResult) { @@ -70,7 +71,7 @@ func RunMigrationAndAssertResult( t.Fatalf("Migration '%s': FAILED\nexpected users %v\nbut got %v", title, expected.Users, actualUsers) } - t.Logf("Migration '%s': PASSED", title) + // t.Logf("Migration '%s': PASSED", title) } func TestMigrate(t *testing.T) { @@ -80,14 +81,13 @@ func TestMigrate(t *testing.T) { } }() - //host := os.Getenv("MONGODB_PORT_27017_TCP_ADDR") - //port := os.Getenv("MONGODB_PORT_27017_TCP_PORT") - host := "127.0.0.1" - port := "27017" + host := os.Getenv("MONGO_PORT_27017_TCP_ADDR") + port := os.Getenv("MONGO_PORT_27017_TCP_PORT") + driverUrl := "mongodb://" + host + ":" + port d0 := driver.GetDriver("mongodb") - d, ok := d0.(*mongodb.MongoDbGoMethodsDriver) + d, ok := d0.(*mongodb.Driver) if !ok { t.Fatal("MongoDbGoMethodsDriver has not registered") } diff --git a/driver/gomethods/mongodb_example/methods_receiver.go b/driver/mongodb/example/sample_mongdb_migrator.go similarity index 66% rename from driver/gomethods/mongodb_example/methods_receiver.go rename to driver/mongodb/example/sample_mongdb_migrator.go index ae96b715..8750969e 100644 --- a/driver/gomethods/mongodb_example/methods_receiver.go +++ b/driver/mongodb/example/sample_mongdb_migrator.go @@ -1,25 +1,26 @@ -package mongodb_example +package example import ( - "github.com/dimag-jfrog/migrate/driver/gomethods" - _ "github.com/dimag-jfrog/migrate/driver/gomethods" - "github.com/dimag-jfrog/migrate/driver/gomethods/mongodb" + "github.com/dimag-jfrog/migrate/driver/mongodb/gomethods" + _ "github.com/dimag-jfrog/migrate/driver/mongodb/gomethods" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" "time" + + "github.com/dimag-jfrog/migrate/driver/mongodb" ) -type MyMgoMethodsReceiver struct { +type SampleMongoDbMigrator struct { } -func (r *MyMgoMethodsReceiver) DbName() string { +func (r *SampleMongoDbMigrator) DbName() string { return DB_NAME } -var _ mongodb.MethodsReceiver = (*MyMgoMethodsReceiver)(nil) +var _ mongodb.MethodsReceiver = (*SampleMongoDbMigrator)(nil) func init() { - gomethods.RegisterMethodsReceiverForDriver("mongodb", &MyMgoMethodsReceiver{}) + gomethods.RegisterMethodsReceiverForDriver("mongodb", &SampleMongoDbMigrator{}) } // Here goes the specific mongodb golang methods driver logic @@ -62,7 +63,7 @@ var UserIds []bson.ObjectId = []bson.ObjectId{ bson.NewObjectId(), } -func (r *MyMgoMethodsReceiver) V001_init_organizations_up(session *mgo.Session) error { +func (r *SampleMongoDbMigrator) V001_init_organizations_up(session *mgo.Session) error { date1, _ := time.Parse(SHORT_DATE_LAYOUT, "1994-Jul-05") date2, _ := time.Parse(SHORT_DATE_LAYOUT, "1998-Sep-04") date3, _ := time.Parse(SHORT_DATE_LAYOUT, "2008-Apr-28") @@ -82,11 +83,11 @@ func (r *MyMgoMethodsReceiver) V001_init_organizations_up(session *mgo.Session) return nil } -func (r *MyMgoMethodsReceiver) V001_init_organizations_down(session *mgo.Session) error { +func (r *SampleMongoDbMigrator) V001_init_organizations_down(session *mgo.Session) error { return session.DB(DB_NAME).C(ORGANIZATIONS_C).DropCollection() } -func (r *MyMgoMethodsReceiver) V001_init_users_up(session *mgo.Session) error { +func (r *SampleMongoDbMigrator) V001_init_users_up(session *mgo.Session) error { users := []User{ {Id: UserIds[0], Name: "Alex"}, {Id: UserIds[1], Name: "Beatrice"}, @@ -102,25 +103,25 @@ func (r *MyMgoMethodsReceiver) V001_init_users_up(session *mgo.Session) error { return nil } -func (r *MyMgoMethodsReceiver) V001_init_users_down(session *mgo.Session) error { +func (r *SampleMongoDbMigrator) V001_init_users_down(session *mgo.Session) error { return session.DB(DB_NAME).C(USERS_C).DropCollection() } -func (r *MyMgoMethodsReceiver) V002_organizations_rename_location_field_to_headquarters_up(session *mgo.Session) error { +func (r *SampleMongoDbMigrator) V002_organizations_rename_location_field_to_headquarters_up(session *mgo.Session) error { c := session.DB(DB_NAME).C(ORGANIZATIONS_C) _, err := c.UpdateAll(nil, bson.M{"$rename": bson.M{"location": "headquarters"}}) return err } -func (r *MyMgoMethodsReceiver) V002_organizations_rename_location_field_to_headquarters_down(session *mgo.Session) error { +func (r *SampleMongoDbMigrator) V002_organizations_rename_location_field_to_headquarters_down(session *mgo.Session) error { c := session.DB(DB_NAME).C(ORGANIZATIONS_C) _, err := c.UpdateAll(nil, bson.M{"$rename": bson.M{"headquarters": "location"}}) return err } -func (r *MyMgoMethodsReceiver) V002_change_user_cleo_to_cleopatra_up(session *mgo.Session) error { +func (r *SampleMongoDbMigrator) V002_change_user_cleo_to_cleopatra_up(session *mgo.Session) error { c := session.DB(DB_NAME).C(USERS_C) colQuerier := bson.M{"name": "Cleo"} @@ -129,7 +130,7 @@ func (r *MyMgoMethodsReceiver) V002_change_user_cleo_to_cleopatra_up(session *mg return c.Update(colQuerier, change) } -func (r *MyMgoMethodsReceiver) V002_change_user_cleo_to_cleopatra_down(session *mgo.Session) error { +func (r *SampleMongoDbMigrator) V002_change_user_cleo_to_cleopatra_down(session *mgo.Session) error { c := session.DB(DB_NAME).C(USERS_C) colQuerier := bson.M{"name": "Cleopatra"} @@ -139,10 +140,10 @@ func (r *MyMgoMethodsReceiver) V002_change_user_cleo_to_cleopatra_down(session * } // Wrong signature methods for testing -func (r *MyMgoMethodsReceiver) V001_method_with_wrong_signature_up(s string) error { +func (r *SampleMongoDbMigrator) V001_method_with_wrong_signature_up(s string) error { return nil } -func (r *MyMgoMethodsReceiver) V001_method_with_wrong_signature_down(session *mgo.Session) (bool, error) { +func (r *SampleMongoDbMigrator) V001_method_with_wrong_signature_down(session *mgo.Session) (bool, error) { return true, nil } diff --git a/driver/gomethods/gomethods_migrator.go b/driver/mongodb/gomethods/gomethods_migrator.go similarity index 96% rename from driver/gomethods/gomethods_migrator.go rename to driver/mongodb/gomethods/gomethods_migrator.go index a6900356..4f99afe2 100644 --- a/driver/gomethods/gomethods_migrator.go +++ b/driver/mongodb/gomethods/gomethods_migrator.go @@ -87,13 +87,6 @@ func (m *Migrator) Migrate(f file.File, pipe chan interface{}) error { return nil } -func reverseInPlace(a []string) { - for i := 0; i < len(a)/2; i++ { - j := len(a) - i - 1 - a[i], a[j] = a[j], a[i] - } -} - func getRollbackToMethod(methodName string) string { if strings.HasSuffix(methodName, "_up") { return strings.TrimSuffix(methodName, "_up") + "_down" diff --git a/driver/gomethods/gomethods_migrator_test.go b/driver/mongodb/gomethods/gomethods_migrator_test.go similarity index 92% rename from driver/gomethods/gomethods_migrator_test.go rename to driver/mongodb/gomethods/gomethods_migrator_test.go index 9dafb8a7..fe6865e0 100644 --- a/driver/gomethods/gomethods_migrator_test.go +++ b/driver/mongodb/gomethods/gomethods_migrator_test.go @@ -220,7 +220,7 @@ func TestMigrate(t *testing.T) { } if !failed { - t.Logf("case '%s': PASSED", c.name) + //t.Logf("case '%s': PASSED", c.name) } } } @@ -245,27 +245,3 @@ func TestGetRollbackToMethod(t *testing.T) { } } } - -func TestReverseInPlace(t *testing.T) { - methods := []string{ - "method1_down", - "method2_down", - "method3_down", - "method4_down", - "method5_down", - } - - expectedReversedMethods := []string{ - "method5_down", - "method4_down", - "method3_down", - "method2_down", - "method1_down", - } - - reverseInPlace(methods) - - if !reflect.DeepEqual(methods, expectedReversedMethods) { - t.Errorf("Expected reverse methods %v but got %v", expectedReversedMethods, methods) - } -} diff --git a/driver/gomethods/gomethods_registry.go b/driver/mongodb/gomethods/gomethods_registry.go similarity index 100% rename from driver/gomethods/gomethods_registry.go rename to driver/mongodb/gomethods/gomethods_registry.go diff --git a/driver/gomethods/mongodb/mongodb.go b/driver/mongodb/mongodb.go similarity index 80% rename from driver/gomethods/mongodb/mongodb.go rename to driver/mongodb/mongodb.go index f07b38b2..ac278728 100644 --- a/driver/gomethods/mongodb/mongodb.go +++ b/driver/mongodb/mongodb.go @@ -3,7 +3,7 @@ package mongodb import ( "errors" "github.com/dimag-jfrog/migrate/driver" - "github.com/dimag-jfrog/migrate/driver/gomethods" + "github.com/dimag-jfrog/migrate/driver/mongodb/gomethods" "github.com/dimag-jfrog/migrate/file" "github.com/dimag-jfrog/migrate/migrate/direction" "gopkg.in/mgo.v2" @@ -27,24 +27,24 @@ func (e WrongMethodsReceiverTypeError) Error() string { const MIGRATE_C = "db_migrations" const DRIVER_NAME = "gomethods.mongodb" -type MongoDbGoMethodsDriver struct { +type Driver struct { Session *mgo.Session methodsReceiver MethodsReceiver migrator gomethods.Migrator } -var _ gomethods.GoMethodsDriver = (*MongoDbGoMethodsDriver)(nil) +var _ gomethods.GoMethodsDriver = (*Driver)(nil) type MethodsReceiver interface { DbName() string } -func (d *MongoDbGoMethodsDriver) MethodsReceiver() interface{} { +func (d *Driver) MethodsReceiver() interface{} { return d.methodsReceiver } -func (d *MongoDbGoMethodsDriver) SetMethodsReceiver(r interface{}) error { +func (d *Driver) SetMethodsReceiver(r interface{}) error { r1, ok := r.(MethodsReceiver) if !ok { return WrongMethodsReceiverTypeError(DRIVER_NAME) @@ -55,7 +55,7 @@ func (d *MongoDbGoMethodsDriver) SetMethodsReceiver(r interface{}) error { } func init() { - driver.RegisterDriver("mongodb", &MongoDbGoMethodsDriver{}) + driver.RegisterDriver("mongodb", &Driver{}) } type DbMigration struct { @@ -63,7 +63,7 @@ type DbMigration struct { Version uint64 `bson:"version"` } -func (driver *MongoDbGoMethodsDriver) Initialize(url string) error { +func (driver *Driver) Initialize(url string) error { if driver.methodsReceiver == nil { return UnregisteredMethodsReceiverError(DRIVER_NAME) } @@ -85,18 +85,18 @@ func (driver *MongoDbGoMethodsDriver) Initialize(url string) error { return nil } -func (driver *MongoDbGoMethodsDriver) Close() error { +func (driver *Driver) Close() error { if driver.Session != nil { driver.Session.Close() } return nil } -func (driver *MongoDbGoMethodsDriver) FilenameExtension() string { +func (driver *Driver) FilenameExtension() string { return "mgo" } -func (driver *MongoDbGoMethodsDriver) Version() (uint64, error) { +func (driver *Driver) Version() (uint64, error) { var latestMigration DbMigration c := driver.Session.DB(driver.methodsReceiver.DbName()).C(MIGRATE_C) @@ -111,7 +111,7 @@ func (driver *MongoDbGoMethodsDriver) Version() (uint64, error) { return latestMigration.Version, nil } } -func (driver *MongoDbGoMethodsDriver) Migrate(f file.File, pipe chan interface{}) { +func (driver *Driver) Migrate(f file.File, pipe chan interface{}) { defer close(pipe) pipe <- f @@ -141,7 +141,7 @@ func (driver *MongoDbGoMethodsDriver) Migrate(f file.File, pipe chan interface{} } } -func (driver *MongoDbGoMethodsDriver) Validate(methodName string) error { +func (driver *Driver) Validate(methodName string) error { method := reflect.ValueOf(driver.methodsReceiver).MethodByName(methodName) if !method.IsValid() { return gomethods.MissingMethodError(methodName) @@ -156,7 +156,7 @@ func (driver *MongoDbGoMethodsDriver) Validate(methodName string) error { return nil } -func (driver *MongoDbGoMethodsDriver) Invoke(methodName string) error { +func (driver *Driver) Invoke(methodName string) error { name := methodName migrateMethod := reflect.ValueOf(driver.methodsReceiver).MethodByName(name) if !migrateMethod.IsValid() { From e9ede5506aca6384312712f24c03bdf9c7bef385 Mon Sep 17 00:00:00 2001 From: dimag Date: Wed, 10 Aug 2016 20:39:41 +0300 Subject: [PATCH 25/42] Added README.md --- driver/mongodb/README.md | 104 ++++++++++++++++++ .../mongodb/example/sample_mongdb_migrator.go | 2 +- 2 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 driver/mongodb/README.md diff --git a/driver/mongodb/README.md b/driver/mongodb/README.md new file mode 100644 index 00000000..fae3ffeb --- /dev/null +++ b/driver/mongodb/README.md @@ -0,0 +1,104 @@ +# MongoDB Driver + +* Runs pre-registered Golang methods that receive a single `*mgo.Session` parameter and return `error` on failure. +* Stores migration version details in collection ``db_migrations``. + This collection will be auto-generated. +* Migrations do not run in transactions, there are no built-in transactions in MongoDB. + That means that if a migration fails, it will not be rolled back. +* There is no out-of-the-box support for command-line interface via terminal. + +## Usage in Go + +```go +import "github.com/dimag-jfrog/migrate/migrate" + +// Import your migration methods package so that they are registered and available for the MongoDB driver. +// There is no need to import the MongoDB driver explicitly, as it should already be imported by your migration methods package. +import _ "my_mongo_db_migrator" + +// use synchronous versions of migration functions ... +allErrors, ok := migrate.UpSync("mongodb://host:port", "./path") +if !ok { + fmt.Println("Oh no ...") + // do sth with allErrors slice +} + +// use the asynchronous version of migration functions ... +pipe := migrate.NewPipe() +go migrate.Up(pipe, "mongodb://host:port", "./path") +// pipe is basically just a channel +// write your own channel listener. see writePipe() in main.go as an example. +``` + +## Migration files format + +The migration files should have an ".mgo" extension and contain a list of registered methods names. + +Migration methods should satisfy the following: +* They should be exported (their name should start with a capital letter) +* Their type should be `func (*mgo.Session) error` + +Recommended (but not required) naming conventions for migration methods: +* Prefix with V : for example V001 for version 1. +* Suffix with "_up" or "_down" for up and down migrations correspondingly. + +001_first_release.up.mgo +``` +V001_some_migration_operation_up +V001_some_other_operation_up +... +``` + +001_first_release.down.mgo +``` +V001_some_other_operation_down +V001_some_migration_operation_down +... +``` + + +## Methods registration + +For a detailed example see: [sample_mongodb_migrator.go](https://github.com/dimag-jfrog/migrate/blob/master/driver/mongodb/example/sample_mongdb_migrator.go) + +```go +package my_mongo_db_migrator + +import ( + "github.com/dimag-jfrog/migrate/driver/mongodb" + "github.com/dimag-jfrog/migrate/driver/mongodb/gomethods" + "gopkg.in/mgo.v2" +) + +// common boilerplate +type MyMongoDbMigrator struct { +} + +func (r *MyMongoDbMigrator) DbName() string { + return "" +} + +var _ mongodb.MethodsReceiver = (*MyMongoDbMigrator)(nil) + +func init() { + gomethods.RegisterMethodsReceiverForDriver("mongodb", &MyMongoDbMigrator{}) +} + + +// Here goes the application-specific migration logic +func (r *MyMongoDbMigrator) V001_some_migration_operation_up(session *mgo.Session) error { + // do something + return nil +} + +func (r *MyMongoDbMigrator) V001_some_migration_operation_down(session *mgo.Session) error { + // revert some_migration_operation_up from above + return nil +} + +``` + +## Authors + +* Demitry Gershovich, https://github.com/dimag-jfrog + diff --git a/driver/mongodb/example/sample_mongdb_migrator.go b/driver/mongodb/example/sample_mongdb_migrator.go index 8750969e..08317778 100644 --- a/driver/mongodb/example/sample_mongdb_migrator.go +++ b/driver/mongodb/example/sample_mongdb_migrator.go @@ -23,7 +23,7 @@ func init() { gomethods.RegisterMethodsReceiverForDriver("mongodb", &SampleMongoDbMigrator{}) } -// Here goes the specific mongodb golang methods driver logic +// Here goes the specific mongodb golang methods migration logic const ( DB_NAME = "test" From 9bd4c8c83c054c22bc3fa1a5832ff5e55627cdc9 Mon Sep 17 00:00:00 2001 From: dimag Date: Wed, 10 Aug 2016 21:25:36 +0300 Subject: [PATCH 26/42] Checking non-exported migration method --- driver/mongodb/example/mongodb_test.go | 21 +++++++++++++++++++ .../mongodb/example/sample_mongdb_migrator.go | 4 ++++ .../mongodb/gomethods/gomethods_migrator.go | 6 ++++++ driver/mongodb/mongodb.go | 10 ++++++--- 4 files changed, 38 insertions(+), 3 deletions(-) diff --git a/driver/mongodb/example/mongodb_test.go b/driver/mongodb/example/mongodb_test.go index c8013d44..33cecb9b 100644 --- a/driver/mongodb/example/mongodb_test.go +++ b/driver/mongodb/example/mongodb_test.go @@ -235,6 +235,27 @@ func TestMigrate(t *testing.T) { Errors: []error{gomethods.MissingMethodError("v001_non_existing_method_up")}, }, }, + { + name: "v0 -> v1: not exported method aborts migration", + file: file.File{ + Path: "/foobar", + FileName: "001_foobar.up.gm", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V001_init_organizations_up + v001_not_exported_method_up + V001_init_users_up + `), + }, + expectedResult: ExpectedMigrationResult{ + Organizations: []Organization{}, + Organizations_v2: []Organization_v2{}, + Users: []User{}, + Errors: []error{gomethods.MethodNotExportedError("v001_not_exported_method_up")}, + }, + }, { name: "v0 -> v1: wrong signature method aborts migration", file: file.File{ diff --git a/driver/mongodb/example/sample_mongdb_migrator.go b/driver/mongodb/example/sample_mongdb_migrator.go index 08317778..f251e5c1 100644 --- a/driver/mongodb/example/sample_mongdb_migrator.go +++ b/driver/mongodb/example/sample_mongdb_migrator.go @@ -140,6 +140,10 @@ func (r *SampleMongoDbMigrator) V002_change_user_cleo_to_cleopatra_down(session } // Wrong signature methods for testing +func (r *SampleMongoDbMigrator) v001_not_exported_method_up(session *mgo.Session) error { + return nil +} + func (r *SampleMongoDbMigrator) V001_method_with_wrong_signature_up(s string) error { return nil } diff --git a/driver/mongodb/gomethods/gomethods_migrator.go b/driver/mongodb/gomethods/gomethods_migrator.go index 4f99afe2..225f757d 100644 --- a/driver/mongodb/gomethods/gomethods_migrator.go +++ b/driver/mongodb/gomethods/gomethods_migrator.go @@ -20,6 +20,12 @@ func (e WrongMethodSignatureError) Error() string { return fmt.Sprintf("Method %s has wrong signature", string(e)) } +type MethodNotExportedError string + +func (e MethodNotExportedError) Error() string { + return fmt.Sprintf("Method %s is not exported", string(e)) +} + type MethodInvocationFailedError struct { MethodName string Err error diff --git a/driver/mongodb/mongodb.go b/driver/mongodb/mongodb.go index ac278728..d077e98b 100644 --- a/driver/mongodb/mongodb.go +++ b/driver/mongodb/mongodb.go @@ -142,14 +142,18 @@ func (driver *Driver) Migrate(f file.File, pipe chan interface{}) { } func (driver *Driver) Validate(methodName string) error { - method := reflect.ValueOf(driver.methodsReceiver).MethodByName(methodName) - if !method.IsValid() { + methodWithReceiver, ok := reflect.TypeOf(driver.methodsReceiver).MethodByName(methodName) + if !ok { return gomethods.MissingMethodError(methodName) } + if methodWithReceiver.PkgPath != "" { + return gomethods.MethodNotExportedError(methodName) + } + methodFunc := reflect.ValueOf(driver.methodsReceiver).MethodByName(methodName) methodTemplate := func(*mgo.Session) error { return nil } - if method.Type() != reflect.TypeOf(methodTemplate) { + if methodFunc.Type() != reflect.TypeOf(methodTemplate) { return gomethods.WrongMethodSignatureError(methodName) } From 32cb8251102e93df2642e0c752757c9b8d2351a6 Mon Sep 17 00:00:00 2001 From: dimag Date: Wed, 10 Aug 2016 21:33:36 +0300 Subject: [PATCH 27/42] specifying version of mongo --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 49684090..a39cf290 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,4 +26,4 @@ mysql: cassandra: image: cassandra:2.2 mongo: - image: mongo + image: mongo:3.2.6 From d4b56e69935d9fb4882015837302c030183c7938 Mon Sep 17 00:00:00 2001 From: dimag Date: Wed, 10 Aug 2016 21:41:52 +0300 Subject: [PATCH 28/42] reverted import path from dimag-jfrog to mattes --- docker-compose.yml | 2 +- driver/bash/bash.go | 4 ++-- driver/cassandra/cassandra.go | 6 +++--- driver/cassandra/cassandra_test.go | 6 +++--- driver/driver.go | 2 +- driver/mongodb/README.md | 8 ++++---- driver/mongodb/example/mongodb_test.go | 12 +++++------ .../mongodb/example/sample_mongdb_migrator.go | 6 +++--- .../mongodb/gomethods/gomethods_migrator.go | 4 ++-- .../gomethods/gomethods_migrator_test.go | 6 +++--- .../mongodb/gomethods/gomethods_registry.go | 2 +- driver/mongodb/mongodb.go | 8 ++++---- driver/mysql/mysql.go | 6 +++--- driver/mysql/mysql_test.go | 6 +++--- driver/postgres/postgres.go | 6 +++--- driver/postgres/postgres_test.go | 6 +++--- driver/sqlite3/sqlite3.go | 6 +++--- driver/sqlite3/sqlite3_test.go | 6 +++--- file/file.go | 2 +- file/file_test.go | 2 +- main.go | 20 +++++++++---------- migrate/migrate.go | 8 ++++---- migrate/migrate_test.go | 4 ++-- 23 files changed, 69 insertions(+), 69 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index a39cf290..01c11e88 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ go: &go image: golang - working_dir: /go/src/github.com/dimag-jfrog/migrate + working_dir: /go/src/github.com/mattes/migrate volumes: - $GOPATH:/go go-test: diff --git a/driver/bash/bash.go b/driver/bash/bash.go index 97ba91f1..031f9bb4 100644 --- a/driver/bash/bash.go +++ b/driver/bash/bash.go @@ -2,8 +2,8 @@ package bash import ( - "github.com/dimag-jfrog/migrate/driver" - "github.com/dimag-jfrog/migrate/file" + "github.com/mattes/migrate/driver" + "github.com/mattes/migrate/file" ) type Driver struct { diff --git a/driver/cassandra/cassandra.go b/driver/cassandra/cassandra.go index ffc76ada..e242de2d 100644 --- a/driver/cassandra/cassandra.go +++ b/driver/cassandra/cassandra.go @@ -9,9 +9,9 @@ import ( "time" "github.com/gocql/gocql" - "github.com/dimag-jfrog/migrate/driver" - "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/migrate/direction" + "github.com/mattes/migrate/driver" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" ) type Driver struct { diff --git a/driver/cassandra/cassandra_test.go b/driver/cassandra/cassandra_test.go index 7d363895..ea737945 100644 --- a/driver/cassandra/cassandra_test.go +++ b/driver/cassandra/cassandra_test.go @@ -7,9 +7,9 @@ import ( "time" "github.com/gocql/gocql" - "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/migrate/direction" - pipep "github.com/dimag-jfrog/migrate/pipe" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" + pipep "github.com/mattes/migrate/pipe" ) func TestMigrate(t *testing.T) { diff --git a/driver/driver.go b/driver/driver.go index 5c2f6fcc..e4ecb783 100644 --- a/driver/driver.go +++ b/driver/driver.go @@ -5,7 +5,7 @@ import ( "fmt" neturl "net/url" // alias to allow `url string` func signature in New - "github.com/dimag-jfrog/migrate/file" + "github.com/mattes/migrate/file" ) // Driver is the interface type that needs to implemented by all drivers. diff --git a/driver/mongodb/README.md b/driver/mongodb/README.md index fae3ffeb..d7d8351d 100644 --- a/driver/mongodb/README.md +++ b/driver/mongodb/README.md @@ -10,7 +10,7 @@ ## Usage in Go ```go -import "github.com/dimag-jfrog/migrate/migrate" +import "github.com/mattes/migrate/migrate" // Import your migration methods package so that they are registered and available for the MongoDB driver. // There is no need to import the MongoDB driver explicitly, as it should already be imported by your migration methods package. @@ -59,14 +59,14 @@ V001_some_migration_operation_down ## Methods registration -For a detailed example see: [sample_mongodb_migrator.go](https://github.com/dimag-jfrog/migrate/blob/master/driver/mongodb/example/sample_mongdb_migrator.go) +For a detailed example see: [sample_mongodb_migrator.go](https://github.com/mattes/migrate/blob/master/driver/mongodb/example/sample_mongdb_migrator.go) ```go package my_mongo_db_migrator import ( - "github.com/dimag-jfrog/migrate/driver/mongodb" - "github.com/dimag-jfrog/migrate/driver/mongodb/gomethods" + "github.com/mattes/migrate/driver/mongodb" + "github.com/mattes/migrate/driver/mongodb/gomethods" "gopkg.in/mgo.v2" ) diff --git a/driver/mongodb/example/mongodb_test.go b/driver/mongodb/example/mongodb_test.go index 33cecb9b..1719b285 100644 --- a/driver/mongodb/example/mongodb_test.go +++ b/driver/mongodb/example/mongodb_test.go @@ -3,13 +3,13 @@ package example import ( "testing" - "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/migrate/direction" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" - "github.com/dimag-jfrog/migrate/driver" - "github.com/dimag-jfrog/migrate/driver/mongodb" - "github.com/dimag-jfrog/migrate/driver/mongodb/gomethods" - pipep "github.com/dimag-jfrog/migrate/pipe" + "github.com/mattes/migrate/driver" + "github.com/mattes/migrate/driver/mongodb" + "github.com/mattes/migrate/driver/mongodb/gomethods" + pipep "github.com/mattes/migrate/pipe" "os" "reflect" "time" diff --git a/driver/mongodb/example/sample_mongdb_migrator.go b/driver/mongodb/example/sample_mongdb_migrator.go index f251e5c1..1ab1440a 100644 --- a/driver/mongodb/example/sample_mongdb_migrator.go +++ b/driver/mongodb/example/sample_mongdb_migrator.go @@ -1,13 +1,13 @@ package example import ( - "github.com/dimag-jfrog/migrate/driver/mongodb/gomethods" - _ "github.com/dimag-jfrog/migrate/driver/mongodb/gomethods" + "github.com/mattes/migrate/driver/mongodb/gomethods" + _ "github.com/mattes/migrate/driver/mongodb/gomethods" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" "time" - "github.com/dimag-jfrog/migrate/driver/mongodb" + "github.com/mattes/migrate/driver/mongodb" ) type SampleMongoDbMigrator struct { diff --git a/driver/mongodb/gomethods/gomethods_migrator.go b/driver/mongodb/gomethods/gomethods_migrator.go index 225f757d..9d0e3f97 100644 --- a/driver/mongodb/gomethods/gomethods_migrator.go +++ b/driver/mongodb/gomethods/gomethods_migrator.go @@ -3,8 +3,8 @@ package gomethods import ( "bufio" "fmt" - "github.com/dimag-jfrog/migrate/driver" - "github.com/dimag-jfrog/migrate/file" + "github.com/mattes/migrate/driver" + "github.com/mattes/migrate/file" "os" "path" "strings" diff --git a/driver/mongodb/gomethods/gomethods_migrator_test.go b/driver/mongodb/gomethods/gomethods_migrator_test.go index fe6865e0..d59dfba3 100644 --- a/driver/mongodb/gomethods/gomethods_migrator_test.go +++ b/driver/mongodb/gomethods/gomethods_migrator_test.go @@ -4,10 +4,10 @@ import ( "reflect" "testing" - "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/migrate/direction" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" - pipep "github.com/dimag-jfrog/migrate/pipe" + pipep "github.com/mattes/migrate/pipe" ) type FakeGoMethodsInvoker struct { diff --git a/driver/mongodb/gomethods/gomethods_registry.go b/driver/mongodb/gomethods/gomethods_registry.go index 5ceb1287..418256fd 100644 --- a/driver/mongodb/gomethods/gomethods_registry.go +++ b/driver/mongodb/gomethods/gomethods_registry.go @@ -2,7 +2,7 @@ package gomethods import ( "fmt" - "github.com/dimag-jfrog/migrate/driver" + "github.com/mattes/migrate/driver" "sync" ) diff --git a/driver/mongodb/mongodb.go b/driver/mongodb/mongodb.go index d077e98b..a60ccc20 100644 --- a/driver/mongodb/mongodb.go +++ b/driver/mongodb/mongodb.go @@ -2,10 +2,10 @@ package mongodb import ( "errors" - "github.com/dimag-jfrog/migrate/driver" - "github.com/dimag-jfrog/migrate/driver/mongodb/gomethods" - "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/migrate/direction" + "github.com/mattes/migrate/driver" + "github.com/mattes/migrate/driver/mongodb/gomethods" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" "reflect" diff --git a/driver/mysql/mysql.go b/driver/mysql/mysql.go index cf186f8f..2bbbc131 100644 --- a/driver/mysql/mysql.go +++ b/driver/mysql/mysql.go @@ -12,9 +12,9 @@ import ( "strings" "github.com/go-sql-driver/mysql" - "github.com/dimag-jfrog/migrate/driver" - "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/migrate/direction" + "github.com/mattes/migrate/driver" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" ) type Driver struct { diff --git a/driver/mysql/mysql_test.go b/driver/mysql/mysql_test.go index 9e08af9f..51cf552a 100644 --- a/driver/mysql/mysql_test.go +++ b/driver/mysql/mysql_test.go @@ -6,9 +6,9 @@ import ( "strings" "testing" - "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/migrate/direction" - pipep "github.com/dimag-jfrog/migrate/pipe" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" + pipep "github.com/mattes/migrate/pipe" ) // TestMigrate runs some additional tests on Migrate(). diff --git a/driver/postgres/postgres.go b/driver/postgres/postgres.go index dbbe4a26..502c7d1a 100644 --- a/driver/postgres/postgres.go +++ b/driver/postgres/postgres.go @@ -8,9 +8,9 @@ import ( "strconv" "github.com/lib/pq" - "github.com/dimag-jfrog/migrate/driver" - "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/migrate/direction" + "github.com/mattes/migrate/driver" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" ) type Driver struct { diff --git a/driver/postgres/postgres_test.go b/driver/postgres/postgres_test.go index 552f23d3..ec91b350 100644 --- a/driver/postgres/postgres_test.go +++ b/driver/postgres/postgres_test.go @@ -5,9 +5,9 @@ import ( "os" "testing" - "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/migrate/direction" - pipep "github.com/dimag-jfrog/migrate/pipe" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" + pipep "github.com/mattes/migrate/pipe" ) // TestMigrate runs some additional tests on Migrate(). diff --git a/driver/sqlite3/sqlite3.go b/driver/sqlite3/sqlite3.go index 65e6a2ac..a92b14d8 100644 --- a/driver/sqlite3/sqlite3.go +++ b/driver/sqlite3/sqlite3.go @@ -7,9 +7,9 @@ import ( "fmt" "strings" - "github.com/dimag-jfrog/migrate/driver" - "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/migrate/direction" + "github.com/mattes/migrate/driver" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" "github.com/mattn/go-sqlite3" ) diff --git a/driver/sqlite3/sqlite3_test.go b/driver/sqlite3/sqlite3_test.go index 29a501f2..8d51a7de 100644 --- a/driver/sqlite3/sqlite3_test.go +++ b/driver/sqlite3/sqlite3_test.go @@ -4,9 +4,9 @@ import ( "database/sql" "testing" - "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/migrate/direction" - pipep "github.com/dimag-jfrog/migrate/pipe" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" + pipep "github.com/mattes/migrate/pipe" ) // TestMigrate runs some additional tests on Migrate() diff --git a/file/file.go b/file/file.go index 14617ba6..54992a48 100644 --- a/file/file.go +++ b/file/file.go @@ -5,7 +5,7 @@ import ( "bytes" "errors" "fmt" - "github.com/dimag-jfrog/migrate/migrate/direction" + "github.com/mattes/migrate/migrate/direction" "go/token" "io/ioutil" "path" diff --git a/file/file_test.go b/file/file_test.go index c6bd0b3c..b9ddeabd 100644 --- a/file/file_test.go +++ b/file/file_test.go @@ -1,7 +1,7 @@ package file import ( - "github.com/dimag-jfrog/migrate/migrate/direction" + "github.com/mattes/migrate/migrate/direction" "io/ioutil" "os" "path" diff --git a/main.go b/main.go index 51d68a84..b6da438f 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,6 @@ // Package main is the CLI. // You can use the CLI via Terminal. -// import "github.com/dimag-jfrog/migrate/migrate" for usage within Go. +// import "github.com/mattes/migrate/migrate" for usage within Go. package main import ( @@ -11,15 +11,15 @@ import ( "time" "github.com/fatih/color" - _ "github.com/dimag-jfrog/migrate/driver/bash" - _ "github.com/dimag-jfrog/migrate/driver/cassandra" - _ "github.com/dimag-jfrog/migrate/driver/mysql" - _ "github.com/dimag-jfrog/migrate/driver/postgres" - _ "github.com/dimag-jfrog/migrate/driver/sqlite3" - "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/migrate" - "github.com/dimag-jfrog/migrate/migrate/direction" - pipep "github.com/dimag-jfrog/migrate/pipe" + _ "github.com/mattes/migrate/driver/bash" + _ "github.com/mattes/migrate/driver/cassandra" + _ "github.com/mattes/migrate/driver/mysql" + _ "github.com/mattes/migrate/driver/postgres" + _ "github.com/mattes/migrate/driver/sqlite3" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate" + "github.com/mattes/migrate/migrate/direction" + pipep "github.com/mattes/migrate/pipe" ) var url = flag.String("url", os.Getenv("MIGRATE_URL"), "") diff --git a/migrate/migrate.go b/migrate/migrate.go index 54cc42bc..b452aeef 100644 --- a/migrate/migrate.go +++ b/migrate/migrate.go @@ -11,10 +11,10 @@ import ( "strconv" "strings" - "github.com/dimag-jfrog/migrate/driver" - "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/migrate/direction" - pipep "github.com/dimag-jfrog/migrate/pipe" + "github.com/mattes/migrate/driver" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" + pipep "github.com/mattes/migrate/pipe" ) // Up applies all available migrations diff --git a/migrate/migrate_test.go b/migrate/migrate_test.go index 7909826d..7d2a622f 100644 --- a/migrate/migrate_test.go +++ b/migrate/migrate_test.go @@ -6,8 +6,8 @@ import ( "testing" // Ensure imports for each driver we wish to test - _ "github.com/dimag-jfrog/migrate/driver/postgres" - _ "github.com/dimag-jfrog/migrate/driver/sqlite3" + _ "github.com/mattes/migrate/driver/postgres" + _ "github.com/mattes/migrate/driver/sqlite3" ) // Add Driver URLs here to test basic Up, Down, .. functions. From 74b161fdb207f9bf4f09eb1eeb1c50204f43ac50 Mon Sep 17 00:00:00 2001 From: Triet Pham Date: Wed, 31 Aug 2016 10:20:16 +0700 Subject: [PATCH 29/42] Add Cassandra url format --- driver/cassandra/README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/driver/cassandra/README.md b/driver/cassandra/README.md index 69596069..75315bfa 100644 --- a/driver/cassandra/README.md +++ b/driver/cassandra/README.md @@ -8,8 +8,13 @@ migrate -url cassandra://host:port/keyspace -path ./db/migrations up migrate help # for more info ``` +Url format +- Authentication: `cassandra://username:password@host:port/keyspace` +- Cassandra v3.x: `cassandra://host:port/keyspace?protocol=4` + + ## Authors * Paul Bergeron, https://github.com/dinedal * Johnny Bergström, https://github.com/balboah -* pateld982, http://github.com/pateld982 \ No newline at end of file +* pateld982, http://github.com/pateld982 From 9a558eed89c0842953ce89dfd4d74a66c25c7a45 Mon Sep 17 00:00:00 2001 From: David Campbell Date: Sun, 23 Oct 2016 14:56:37 -0700 Subject: [PATCH 30/42] Skip tests requiring a DB connection with the '-short' flag - go-test target for local testing --- Makefile | 14 +++++++++++--- driver/cassandra/cassandra_test.go | 4 ++++ driver/mysql/mysql_test.go | 3 +++ driver/postgres/postgres_test.go | 4 ++++ driver/sqlite3/sqlite3_test.go | 4 ++++ migrate/migrate_test.go | 18 ++++++++++++++++++ 6 files changed, 44 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 9cee255b..8e46968d 100644 --- a/Makefile +++ b/Makefile @@ -1,14 +1,22 @@ +TESTFLAGS?= IMAGE=mattes/migrate DCR=docker-compose run --rm -.PHONY: clean test build release docker-build docker-push run +GOTEST=go test $(TESTFLAGS) `go list ./... | grep -v "/vendor/"` +.PHONY: clean test build release docker-build docker-push run all: release clean: rm -f migrate -test: - $(DCR) go-test +fmt: + @gofmt -s -w `go list -f {{.Dir}} ./... | grep -v "/vendor/"` + +test: fmt + $(DCR) $(GOTEST) + +go-test: fmt + @$(GOTEST) build: $(DCR) go-build diff --git a/driver/cassandra/cassandra_test.go b/driver/cassandra/cassandra_test.go index ea737945..c0684314 100644 --- a/driver/cassandra/cassandra_test.go +++ b/driver/cassandra/cassandra_test.go @@ -13,6 +13,10 @@ import ( ) func TestMigrate(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } + var session *gocql.Session host := os.Getenv("CASSANDRA_PORT_9042_TCP_ADDR") diff --git a/driver/mysql/mysql_test.go b/driver/mysql/mysql_test.go index 51cf552a..ec192716 100644 --- a/driver/mysql/mysql_test.go +++ b/driver/mysql/mysql_test.go @@ -14,6 +14,9 @@ import ( // TestMigrate runs some additional tests on Migrate(). // Basic testing is already done in migrate/migrate_test.go func TestMigrate(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } host := os.Getenv("MYSQL_PORT_3306_TCP_ADDR") port := os.Getenv("MYSQL_PORT_3306_TCP_PORT") driverUrl := "mysql://root@tcp(" + host + ":" + port + ")/migratetest" diff --git a/driver/postgres/postgres_test.go b/driver/postgres/postgres_test.go index ec91b350..a1c04958 100644 --- a/driver/postgres/postgres_test.go +++ b/driver/postgres/postgres_test.go @@ -13,6 +13,10 @@ import ( // TestMigrate runs some additional tests on Migrate(). // Basic testing is already done in migrate/migrate_test.go func TestMigrate(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } + host := os.Getenv("POSTGRES_PORT_5432_TCP_ADDR") port := os.Getenv("POSTGRES_PORT_5432_TCP_PORT") driverUrl := "postgres://postgres@" + host + ":" + port + "/template1?sslmode=disable" diff --git a/driver/sqlite3/sqlite3_test.go b/driver/sqlite3/sqlite3_test.go index 8d51a7de..7f50095a 100644 --- a/driver/sqlite3/sqlite3_test.go +++ b/driver/sqlite3/sqlite3_test.go @@ -12,6 +12,10 @@ import ( // TestMigrate runs some additional tests on Migrate() // Basic testing is already done in migrate/migrate_test.go func TestMigrate(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } + driverFile := ":memory:" driverUrl := "sqlite3://" + driverFile diff --git a/migrate/migrate_test.go b/migrate/migrate_test.go index 7d2a622f..85492689 100644 --- a/migrate/migrate_test.go +++ b/migrate/migrate_test.go @@ -16,6 +16,9 @@ var driverUrls = []string{ } func TestCreate(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } for _, driverUrl := range driverUrls { t.Logf("Test driver: %s", driverUrl) tmpdir, err := ioutil.TempDir("/tmp", "migrate-test") @@ -57,6 +60,9 @@ func TestCreate(t *testing.T) { } func TestReset(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } for _, driverUrl := range driverUrls { t.Logf("Test driver: %s", driverUrl) tmpdir, err := ioutil.TempDir("/", "migrate-test") @@ -82,6 +88,9 @@ func TestReset(t *testing.T) { } func TestDown(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } for _, driverUrl := range driverUrls { t.Logf("Test driver: %s", driverUrl) tmpdir, err := ioutil.TempDir("/tmp", "migrate-test") @@ -119,6 +128,9 @@ func TestDown(t *testing.T) { } func TestUp(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } for _, driverUrl := range driverUrls { t.Logf("Test driver: %s", driverUrl) tmpdir, err := ioutil.TempDir("/tmp", "migrate-test") @@ -156,6 +168,9 @@ func TestUp(t *testing.T) { } func TestRedo(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } for _, driverUrl := range driverUrls { t.Logf("Test driver: %s", driverUrl) tmpdir, err := ioutil.TempDir("/tmp", "migrate-test") @@ -193,6 +208,9 @@ func TestRedo(t *testing.T) { } func TestMigrate(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } for _, driverUrl := range driverUrls { t.Logf("Test driver: %s", driverUrl) tmpdir, err := ioutil.TempDir("/tmp", "migrate-test") From 04e1cb0cfcb403499ef0ba45785a36d09e784928 Mon Sep 17 00:00:00 2001 From: David Campbell Date: Sun, 23 Oct 2016 15:05:54 -0700 Subject: [PATCH 31/42] Bump travis versions to use Go 1.6 and 1.7 --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 512b33a3..6e027af6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,8 @@ sudo: required go: - 1.4 - 1.5 + - 1.6 + - 1.7 services: - docker From d290bf94606b42b59cbf3745669b75b0a550efb2 Mon Sep 17 00:00:00 2001 From: David Campbell Date: Sun, 23 Oct 2016 15:11:04 -0700 Subject: [PATCH 32/42] - docker command should remain 'go-test' --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8e46968d..eaf7e122 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ fmt: @gofmt -s -w `go list -f {{.Dir}} ./... | grep -v "/vendor/"` test: fmt - $(DCR) $(GOTEST) + $(DCR) go-test go-test: fmt @$(GOTEST) From 478fb42df5bd7d6ec358ed3f4323b149224f7792 Mon Sep 17 00:00:00 2001 From: Matthias Kadenbach Date: Wed, 21 Dec 2016 20:13:57 -0800 Subject: [PATCH 33/42] Update README.md --- driver/postgres/README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/driver/postgres/README.md b/driver/postgres/README.md index b7d12b74..e5b1b846 100644 --- a/driver/postgres/README.md +++ b/driver/postgres/README.md @@ -15,9 +15,11 @@ migrate -url postgres://user@host:port/database -path ./db/migrations up migrate help # for more info # TODO(mattes): thinking about adding some custom flag to allow migration within schemas: --url="postgres://user@host:port/database?schema=name" +-url="postgres://user@host:port/database?schema=name" + +# see more docs: https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parameters ``` ## Authors -* Matthias Kadenbach, https://github.com/mattes \ No newline at end of file +* Matthias Kadenbach, https://github.com/mattes From d4796c81042d8c1caf10b2353a2e3e98915fdbb8 Mon Sep 17 00:00:00 2001 From: David Campbell Date: Thu, 22 Dec 2016 12:30:34 -0800 Subject: [PATCH 34/42] Update README to reflect epoch filenames --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 415a3aff..f103123a 100644 --- a/README.md +++ b/README.md @@ -100,15 +100,15 @@ go migrate.Up(pipe, "driver://url", "./path") The format of migration files looks like this: ``` -001_initial_plan_to_do_sth.up.sql # up migration instructions -001_initial_plan_to_do_sth.down.sql # down migration instructions -002_xxx.up.sql -002_xxx.down.sql +1481574547_initial_plan_to_do_sth.up.sql # up migration instructions +1481574547_initial_plan_to_do_sth.down.sql # down migration instructions +1482438365_xxx.up.sql +1482438365_xxx.down.sql ... ``` Why two files? This way you could still do sth like -``psql -f ./db/migrations/001_initial_plan_to_do_sth.up.sql`` and there is no +``psql -f ./db/migrations/1481574547_initial_plan_to_do_sth.up.sql`` and there is no need for any custom markup language to divide up and down migrations. Please note that the filename extension depends on the driver. From 5773f756982529fdf082fa234e8fa04541a3e645 Mon Sep 17 00:00:00 2001 From: Sam Willcocks Date: Thu, 5 Jan 2017 15:09:41 +0000 Subject: [PATCH 35/42] Add Ql driver Adds a driver for the embedded Ql database (https://github.com/cznic/ql) --- README.md | 1 + driver/ql/README.md | 17 ++++++ driver/ql/ql.go | 130 ++++++++++++++++++++++++++++++++++++++++ main.go | 1 + migrate/migrate_test.go | 2 + 5 files changed, 151 insertions(+) create mode 100644 driver/ql/README.md create mode 100644 driver/ql/ql.go diff --git a/README.md b/README.md index f103123a..bb4dc738 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ __Features__ * [Cassandra](https://github.com/mattes/migrate/tree/master/driver/cassandra) * [SQLite](https://github.com/mattes/migrate/tree/master/driver/sqlite3) * [MySQL](https://github.com/mattes/migrate/tree/master/driver/mysql) ([experimental](https://github.com/mattes/migrate/issues/1#issuecomment-58728186)) + * [Ql](https://github.com/mattes/migrate/tree/master/driver/ql) * Bash (planned) Need another driver? Just implement the [Driver interface](http://godoc.org/github.com/mattes/migrate/driver#Driver) and open a PR. diff --git a/driver/ql/README.md b/driver/ql/README.md new file mode 100644 index 00000000..de53f814 --- /dev/null +++ b/driver/ql/README.md @@ -0,0 +1,17 @@ +# Ql Driver + +* Supports both in-memory and file ql databases, with the URL schemes `ql+file://` and `ql+memory://` + * In-memory driver is not of much use, since the database is destroyed once the migration operation finishes, but is included for completeness. +* Stores migration version details in automatically generated table ``schema_migrations``. + +## Usage + +```bash +migrate -url ql+file://./data.db -path ./migrations create add_field_to_table +migrate -url ql+file://./data.db -path ./db/migrations up +migrate help # for more info +``` + +## Authors + +* Sam Willcocks, https://github.com/wlcx diff --git a/driver/ql/ql.go b/driver/ql/ql.go new file mode 100644 index 00000000..1c75fe70 --- /dev/null +++ b/driver/ql/ql.go @@ -0,0 +1,130 @@ +package ql + +import ( + "database/sql" + + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/driver" + "github.com/mattes/migrate/migrate/direction" + + _ "github.com/cznic/ql/driver" + "strings" + "fmt" +) + +const tableName = "schema_migration" + +type Driver struct { + db *sql.DB +} + +func (d *Driver) Initialize(url string) (err error) { + d.db, err = sql.Open("ql", strings.TrimPrefix(url, "ql+")) + if err != nil { + return + } + if err = d.db.Ping(); err != nil { + return + } + if err = d.ensureVersionTableExists(); err != nil { + return + } + return +} + +func (d *Driver) Close() error { + if err := d.db.Close(); err != nil { + return err + } + return nil +} + +func (d *Driver) FilenameExtension() string { + return "sql" +} + +func (d *Driver) Migrate(f file.File, pipe chan interface{}) { + defer close(pipe) + pipe <- f + + tx, err := d.db.Begin() + if err != nil { + pipe <- err + return + } + + switch f.Direction { + case direction.Up: + if _, err := tx.Exec("INSERT INTO "+tableName+" (version) VALUES (uint($1))", f.Version); err != nil { + pipe <- err + if err := tx.Rollback(); err != nil { + pipe <- err + } + return + } + case direction.Down: + if _, err := tx.Exec("DELETE FROM "+tableName+" WHERE version=uint($1)", f.Version); err != nil { + pipe <- err + if err := tx.Rollback(); err != nil { + pipe <- err + } + return + } + } + + if err := f.ReadContent(); err != nil { + pipe <- err + return + } + + if _, err := tx.Exec(string(f.Content)); err != nil { + pipe <- err + if err := tx.Rollback(); err != nil { + pipe <- err + } + return + } + + if err := tx.Commit(); err != nil { + pipe <- err + return + } +} + +func (d *Driver) Version() (uint64, error) { + var version uint64 + err := d.db.QueryRow("SELECT version FROM " + tableName + " ORDER BY version DESC LIMIT 1").Scan(&version) + switch { + case err == sql.ErrNoRows: + return 0, nil + case err != nil: + return 0, err + default: + return version, nil + } +} + +func (d *Driver) ensureVersionTableExists() error { + tx, err := d.db.Begin() + if err != nil { + return err + } + if _, err := tx.Exec(fmt.Sprintf(` + CREATE TABLE IF NOT EXISTS %s (version uint64); + CREATE UNIQUE INDEX IF NOT EXISTS version_unique ON %s (version); + `, tableName, tableName)); err != nil { + if err := tx.Rollback(); err != nil { + return err + } + return err + } + if err := tx.Commit(); err != nil { + return err + } + return nil +} + +func init() { + driver.RegisterDriver("ql+file", &Driver{}) + driver.RegisterDriver("ql+memory", &Driver{}) +} \ No newline at end of file diff --git a/main.go b/main.go index d50671fa..ac842c7d 100644 --- a/main.go +++ b/main.go @@ -17,6 +17,7 @@ import ( _ "github.com/mattes/migrate/driver/mysql" _ "github.com/mattes/migrate/driver/postgres" _ "github.com/mattes/migrate/driver/sqlite3" + _ "github.com/mattes/migrate/driver/ql" "github.com/mattes/migrate/file" "github.com/mattes/migrate/migrate" "github.com/mattes/migrate/migrate/direction" diff --git a/migrate/migrate_test.go b/migrate/migrate_test.go index c55655f3..c7b43adb 100644 --- a/migrate/migrate_test.go +++ b/migrate/migrate_test.go @@ -10,11 +10,13 @@ import ( _ "github.com/mattes/migrate/driver/postgres" _ "github.com/mattes/migrate/driver/sqlite3" + _ "github.com/mattes/migrate/driver/ql" ) // Add Driver URLs here to test basic Up, Down, .. functions. var driverUrls = []string{ "postgres://postgres@" + os.Getenv("POSTGRES_PORT_5432_TCP_ADDR") + ":" + os.Getenv("POSTGRES_PORT_5432_TCP_PORT") + "/template1?sslmode=disable", + "ql+file://./test.db", } func tearDown(driverUrl, tmpdir string) { From 660297c49102f63c4f8a73a698e03ea4f7399140 Mon Sep 17 00:00:00 2001 From: dimag Date: Fri, 6 Jan 2017 17:11:24 +0200 Subject: [PATCH 36/42] - Merge branch 'prepare_pr' of https://github.com/dimag-jfrog/migrate - Fixed test with not-exported method identified as missing: unified the 2 errors into one - Fixed error message formatting # Conflicts: # docker-compose.yml --- driver/mongodb/example/mongodb_test.go | 5 +++-- driver/mongodb/gomethods/gomethods_migrator.go | 14 +++++--------- .../mongodb/gomethods/gomethods_migrator_test.go | 4 ++-- driver/mongodb/mongodb.go | 6 +++--- driver/ql/ql.go | 6 +++--- main.go | 2 +- migrate/migrate_test.go | 2 +- 7 files changed, 18 insertions(+), 21 deletions(-) diff --git a/driver/mongodb/example/mongodb_test.go b/driver/mongodb/example/mongodb_test.go index 1719b285..3f9cdfae 100644 --- a/driver/mongodb/example/mongodb_test.go +++ b/driver/mongodb/example/mongodb_test.go @@ -232,7 +232,7 @@ func TestMigrate(t *testing.T) { Organizations: []Organization{}, Organizations_v2: []Organization_v2{}, Users: []User{}, - Errors: []error{gomethods.MissingMethodError("v001_non_existing_method_up")}, + Errors: []error{gomethods.MethodNotFoundError("v001_non_existing_method_up")}, }, }, { @@ -253,7 +253,8 @@ func TestMigrate(t *testing.T) { Organizations: []Organization{}, Organizations_v2: []Organization_v2{}, Users: []User{}, - Errors: []error{gomethods.MethodNotExportedError("v001_not_exported_method_up")}, + //Errors: []error{gomethods.MethodNotExportedError("v001_not_exported_method_up")}, + Errors: []error{gomethods.MethodNotFoundError("v001_not_exported_method_up")}, }, }, { diff --git a/driver/mongodb/gomethods/gomethods_migrator.go b/driver/mongodb/gomethods/gomethods_migrator.go index 9d0e3f97..81ac6c18 100644 --- a/driver/mongodb/gomethods/gomethods_migrator.go +++ b/driver/mongodb/gomethods/gomethods_migrator.go @@ -10,20 +10,16 @@ import ( "strings" ) -type MissingMethodError string - -func (e MissingMethodError) Error() string { return "Non existing migrate method: " + string(e) } - type WrongMethodSignatureError string func (e WrongMethodSignatureError) Error() string { - return fmt.Sprintf("Method %s has wrong signature", string(e)) + return fmt.Sprintf("Migrate method '%s' has a wrong signature", string(e)) } -type MethodNotExportedError string +type MethodNotFoundError string -func (e MethodNotExportedError) Error() string { - return fmt.Sprintf("Method %s is not exported", string(e)) +func (e MethodNotFoundError) Error() string { + return fmt.Sprintf("Migrate method '%s' was not found. It is either non existing or has not been exported (starts with lowercase).", string(e)) } type MethodInvocationFailedError struct { @@ -32,7 +28,7 @@ type MethodInvocationFailedError struct { } func (e *MethodInvocationFailedError) Error() string { - return fmt.Sprintf("Method %s returned an error: %v", e.MethodName, e.Error) + return fmt.Sprintf("Method '%s' returned an error: %v", e.MethodName, e.Err) } type MigrationMethodInvoker interface { diff --git a/driver/mongodb/gomethods/gomethods_migrator_test.go b/driver/mongodb/gomethods/gomethods_migrator_test.go index d59dfba3..d94506ee 100644 --- a/driver/mongodb/gomethods/gomethods_migrator_test.go +++ b/driver/mongodb/gomethods/gomethods_migrator_test.go @@ -16,7 +16,7 @@ type FakeGoMethodsInvoker struct { func (invoker *FakeGoMethodsInvoker) Validate(methodName string) error { if methodName == "V001_some_non_existing_method_up" { - return MissingMethodError(methodName) + return MethodNotFoundError(methodName) } return nil @@ -93,7 +93,7 @@ func TestMigrate(t *testing.T) { `), }, expectedInvokedMethods: []string{}, - expectedErrors: []error{MissingMethodError("V001_some_non_existing_method_up")}, + expectedErrors: []error{MethodNotFoundError("V001_some_non_existing_method_up")}, }, { name: "up migration: failing method stops execution", diff --git a/driver/mongodb/mongodb.go b/driver/mongodb/mongodb.go index a60ccc20..292506d3 100644 --- a/driver/mongodb/mongodb.go +++ b/driver/mongodb/mongodb.go @@ -144,10 +144,10 @@ func (driver *Driver) Migrate(f file.File, pipe chan interface{}) { func (driver *Driver) Validate(methodName string) error { methodWithReceiver, ok := reflect.TypeOf(driver.methodsReceiver).MethodByName(methodName) if !ok { - return gomethods.MissingMethodError(methodName) + return gomethods.MethodNotFoundError(methodName) } if methodWithReceiver.PkgPath != "" { - return gomethods.MethodNotExportedError(methodName) + return gomethods.MethodNotFoundError(methodName) } methodFunc := reflect.ValueOf(driver.methodsReceiver).MethodByName(methodName) @@ -164,7 +164,7 @@ func (driver *Driver) Invoke(methodName string) error { name := methodName migrateMethod := reflect.ValueOf(driver.methodsReceiver).MethodByName(name) if !migrateMethod.IsValid() { - return gomethods.MissingMethodError(methodName) + return gomethods.MethodNotFoundError(methodName) } retValues := migrateMethod.Call([]reflect.Value{reflect.ValueOf(driver.Session)}) diff --git a/driver/ql/ql.go b/driver/ql/ql.go index 1c75fe70..3250b63a 100644 --- a/driver/ql/ql.go +++ b/driver/ql/ql.go @@ -3,13 +3,13 @@ package ql import ( "database/sql" - "github.com/mattes/migrate/file" "github.com/mattes/migrate/driver" + "github.com/mattes/migrate/file" "github.com/mattes/migrate/migrate/direction" + "fmt" _ "github.com/cznic/ql/driver" "strings" - "fmt" ) const tableName = "schema_migration" @@ -127,4 +127,4 @@ func (d *Driver) ensureVersionTableExists() error { func init() { driver.RegisterDriver("ql+file", &Driver{}) driver.RegisterDriver("ql+memory", &Driver{}) -} \ No newline at end of file +} diff --git a/main.go b/main.go index ac842c7d..066b102b 100644 --- a/main.go +++ b/main.go @@ -16,8 +16,8 @@ import ( _ "github.com/mattes/migrate/driver/crate" _ "github.com/mattes/migrate/driver/mysql" _ "github.com/mattes/migrate/driver/postgres" - _ "github.com/mattes/migrate/driver/sqlite3" _ "github.com/mattes/migrate/driver/ql" + _ "github.com/mattes/migrate/driver/sqlite3" "github.com/mattes/migrate/file" "github.com/mattes/migrate/migrate" "github.com/mattes/migrate/migrate/direction" diff --git a/migrate/migrate_test.go b/migrate/migrate_test.go index c7b43adb..e202a61e 100644 --- a/migrate/migrate_test.go +++ b/migrate/migrate_test.go @@ -9,8 +9,8 @@ import ( // Ensure imports for each driver we wish to test _ "github.com/mattes/migrate/driver/postgres" - _ "github.com/mattes/migrate/driver/sqlite3" _ "github.com/mattes/migrate/driver/ql" + _ "github.com/mattes/migrate/driver/sqlite3" ) // Add Driver URLs here to test basic Up, Down, .. functions. From 739d0b221bfb37fa82c540cf80f23247bf86e670 Mon Sep 17 00:00:00 2001 From: dimag Date: Fri, 6 Jan 2017 17:58:04 +0200 Subject: [PATCH 37/42] Ensuring uniqueness of the "version" field --- driver/mongodb/mongodb.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/driver/mongodb/mongodb.go b/driver/mongodb/mongodb.go index 292506d3..a9fe9a63 100644 --- a/driver/mongodb/mongodb.go +++ b/driver/mongodb/mongodb.go @@ -79,6 +79,15 @@ func (driver *Driver) Initialize(url string) error { } session.SetMode(mgo.Monotonic, true) + c := session.DB(driver.methodsReceiver.DbName()).C(MIGRATE_C) + err = c.EnsureIndex(mgo.Index{ + Key: []string{"version"}, + Unique: true, + }) + if err != nil { + return err + } + driver.Session = session driver.migrator = gomethods.Migrator{MethodInvoker: driver} From 686d4f0ea219fa3c366aaec918389d334d849e02 Mon Sep 17 00:00:00 2001 From: David Campbell Date: Tue, 15 Dec 2015 17:57:16 -0800 Subject: [PATCH 38/42] prefix migration files with epoch Using epoch prefixed filenames, multiple developers can commit migrations while minimizing versioning conflicts. This commit remains backwards compatible with all previous migrations and requires no changes to the `schema_migrations` table. This commit also cleans up migrate-test temp directories --- migrate/migrate_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/migrate/migrate_test.go b/migrate/migrate_test.go index e202a61e..be1c5591 100644 --- a/migrate/migrate_test.go +++ b/migrate/migrate_test.go @@ -24,6 +24,11 @@ func tearDown(driverUrl, tmpdir string) { os.RemoveAll(tmpdir) } +func tearDown(driverUrl, tmpdir string) { + DownSync(driverUrl, tmpdir) + os.RemoveAll(tmpdir) +} + func TestCreate(t *testing.T) { if testing.Short() { t.Skip("skipping test in short mode.") From 283effdfb00569a68837685c86d55797c4e8e18f Mon Sep 17 00:00:00 2001 From: Till Klocke Date: Thu, 16 Jun 2016 11:48:39 +0200 Subject: [PATCH 39/42] Added driver for crate.io database --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index 1c34df6b..8dcd17c5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,3 +31,4 @@ crate: mongo: image: mongo:3.2.6 + From 948c06b256f4fa030ca87cd9e364f6225d8fd689 Mon Sep 17 00:00:00 2001 From: Sam Willcocks Date: Thu, 5 Jan 2017 15:09:41 +0000 Subject: [PATCH 40/42] Add Ql driver Adds a driver for the embedded Ql database (https://github.com/cznic/ql) --- main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/main.go b/main.go index 066b102b..e3ee644d 100644 --- a/main.go +++ b/main.go @@ -18,6 +18,7 @@ import ( _ "github.com/mattes/migrate/driver/postgres" _ "github.com/mattes/migrate/driver/ql" _ "github.com/mattes/migrate/driver/sqlite3" + _ "github.com/mattes/migrate/driver/ql" "github.com/mattes/migrate/file" "github.com/mattes/migrate/migrate" "github.com/mattes/migrate/migrate/direction" From ecab7c3870c381901a46f4badd724f1790949271 Mon Sep 17 00:00:00 2001 From: dimag Date: Sat, 7 Jan 2017 01:17:19 +0200 Subject: [PATCH 41/42] Reverted the merge, reapplied changes and rebased master --- driver/mongodb/example/mongodb_test.go | 3 +-- driver/mongodb/gomethods/gomethods_migrator.go | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/driver/mongodb/example/mongodb_test.go b/driver/mongodb/example/mongodb_test.go index 3f9cdfae..86c37c00 100644 --- a/driver/mongodb/example/mongodb_test.go +++ b/driver/mongodb/example/mongodb_test.go @@ -253,8 +253,7 @@ func TestMigrate(t *testing.T) { Organizations: []Organization{}, Organizations_v2: []Organization_v2{}, Users: []User{}, - //Errors: []error{gomethods.MethodNotExportedError("v001_not_exported_method_up")}, - Errors: []error{gomethods.MethodNotFoundError("v001_not_exported_method_up")}, + Errors: []error{gomethods.MethodNotFoundError("v001_not_exported_method_up")}, }, }, { diff --git a/driver/mongodb/gomethods/gomethods_migrator.go b/driver/mongodb/gomethods/gomethods_migrator.go index 81ac6c18..797c6f29 100644 --- a/driver/mongodb/gomethods/gomethods_migrator.go +++ b/driver/mongodb/gomethods/gomethods_migrator.go @@ -13,13 +13,13 @@ import ( type WrongMethodSignatureError string func (e WrongMethodSignatureError) Error() string { - return fmt.Sprintf("Migrate method '%s' has a wrong signature", string(e)) + return fmt.Sprintf("Method '%s' has wrong signature", string(e)) } type MethodNotFoundError string func (e MethodNotFoundError) Error() string { - return fmt.Sprintf("Migrate method '%s' was not found. It is either non existing or has not been exported (starts with lowercase).", string(e)) + return fmt.Sprintf("Method '%s' was not found. It is either non-existing or has not been exported (starts with lowercase).", string(e)) } type MethodInvocationFailedError struct { From 2d7a79d140675bda494baf8b208e5ddbacf2359d Mon Sep 17 00:00:00 2001 From: dimag Date: Sat, 7 Jan 2017 01:55:07 +0200 Subject: [PATCH 42/42] fixed errors that were introduced in last merge-rebase --- driver/ql/ql.go | 6 +++--- main.go | 1 - migrate/migrate_test.go | 5 ----- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/driver/ql/ql.go b/driver/ql/ql.go index 3250b63a..1c75fe70 100644 --- a/driver/ql/ql.go +++ b/driver/ql/ql.go @@ -3,13 +3,13 @@ package ql import ( "database/sql" - "github.com/mattes/migrate/driver" "github.com/mattes/migrate/file" + "github.com/mattes/migrate/driver" "github.com/mattes/migrate/migrate/direction" - "fmt" _ "github.com/cznic/ql/driver" "strings" + "fmt" ) const tableName = "schema_migration" @@ -127,4 +127,4 @@ func (d *Driver) ensureVersionTableExists() error { func init() { driver.RegisterDriver("ql+file", &Driver{}) driver.RegisterDriver("ql+memory", &Driver{}) -} +} \ No newline at end of file diff --git a/main.go b/main.go index e3ee644d..066b102b 100644 --- a/main.go +++ b/main.go @@ -18,7 +18,6 @@ import ( _ "github.com/mattes/migrate/driver/postgres" _ "github.com/mattes/migrate/driver/ql" _ "github.com/mattes/migrate/driver/sqlite3" - _ "github.com/mattes/migrate/driver/ql" "github.com/mattes/migrate/file" "github.com/mattes/migrate/migrate" "github.com/mattes/migrate/migrate/direction" diff --git a/migrate/migrate_test.go b/migrate/migrate_test.go index be1c5591..e202a61e 100644 --- a/migrate/migrate_test.go +++ b/migrate/migrate_test.go @@ -24,11 +24,6 @@ func tearDown(driverUrl, tmpdir string) { os.RemoveAll(tmpdir) } -func tearDown(driverUrl, tmpdir string) { - DownSync(driverUrl, tmpdir) - os.RemoveAll(tmpdir) -} - func TestCreate(t *testing.T) { if testing.Short() { t.Skip("skipping test in short mode.")