diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/README.md b/README.md index 9d45be5..06a1551 100644 --- a/README.md +++ b/README.md @@ -1 +1,228 @@ -gorm-goose +# gorm-goose + +This is a fork of [https://bitbucket.org/liamstask/goose](https://bitbucket.org/liamstask/goose) for [gorm](https://github.com/jinzhu/gorm). + +gorm-goose is a database migration tool for [gorm](https://github.com/jinzhu/gorm). +Currently, available drivers are: "postgres", "mysql", or "sqlite3". + +You can manage your database's evolution by creating incremental SQL or Go scripts. + +# Install + + $ go get github.com/Altoros/gorm-goose/cmd/gorm-goose + +This will install the `gorm-goose` binary to your `$GOPATH/bin` directory. + +You can also build gorm-goose into your own applications by importing `github.com/Altoros/gorm-goose/lib/goose`. + +# Usage + +gorm-goose provides several commands to help manage your database schema. + +## create + +Create a new Go migration. + + $ gorm-goose create AddSomeColumns + $ goose: created db/migrations/20130106093224_AddSomeColumns.go + +Edit the newly created script to define the behavior of your migration. + +You can also create an SQL migration: + + $ gorm-goose create AddSomeColumns sql + $ goose: created db/migrations/20130106093224_AddSomeColumns.sql + +## up + +Apply all available migrations. + + $ gorm-goose up + $ goose: migrating db environment 'development', current version: 0, target: 3 + $ OK 001_basics.sql + $ OK 002_next.sql + $ OK 003_and_again.go + +### option: pgschema + +Use the `pgschema` flag with the `up` command specify a postgres schema. + + $ gorm-goose -pgschema=my_schema_name up + $ goose: migrating db environment 'development', current version: 0, target: 3 + $ OK 001_basics.sql + $ OK 002_next.sql + $ OK 003_and_again.go + +## down + +Roll back a single migration from the current version. + + $ gorm-goose down + $ goose: migrating db environment 'development', current version: 3, target: 2 + $ OK 003_and_again.go + +## redo + +Roll back the most recently applied migration, then run it again. + + $ gorm-goose redo + $ goose: migrating db environment 'development', current version: 3, target: 2 + $ OK 003_and_again.go + $ goose: migrating db environment 'development', current version: 2, target: 3 + $ OK 003_and_again.go + +## status + +Print the status of all migrations: + + $ gorm-goose status + $ goose: status for environment 'development' + $ Applied At Migration + $ ======================================= + $ Sun Jan 6 11:25:03 2013 -- 001_basics.sql + $ Sun Jan 6 11:25:03 2013 -- 002_next.sql + $ Pending -- 003_and_again.go + +## dbversion + +Print the current version of the database: + + $ gorm-goose dbversion + $ goose: dbversion 002 + + +`gorm-goose -h` provides more detailed info on each command. + + +# Migrations + +gorm-goose supports migrations written in SQL or in Go - see the `gorm-goose create` command above for details on how to generate them. + +## SQL Migrations + +A sample SQL migration looks like: + +```sql +-- +goose Up +CREATE TABLE post ( + id int NOT NULL, + title text, + body text, + PRIMARY KEY(id) +); + +-- +goose Down +DROP TABLE post; +``` + +Notice the annotations in the comments. Any statements following `-- +goose Up` will be executed as part of a forward migration, and any statements following `-- +goose Down` will be executed as part of a rollback. + +By default, SQL statements are delimited by semicolons - in fact, query statements must end with a semicolon to be properly recognized by goose. + +More complex statements (PL/pgSQL) that have semicolons within them must be annotated with `-- +goose StatementBegin` and `-- +goose StatementEnd` to be properly recognized. For example: + +```sql +-- +goose Up +-- +goose StatementBegin +CREATE OR REPLACE FUNCTION histories_partition_creation( DATE, DATE ) +returns void AS $$ +DECLARE + create_query text; +BEGIN + FOR create_query IN SELECT + 'CREATE TABLE IF NOT EXISTS histories_' + || TO_CHAR( d, 'YYYY_MM' ) + || ' ( CHECK( created_at >= timestamp ''' + || TO_CHAR( d, 'YYYY-MM-DD 00:00:00' ) + || ''' AND created_at < timestamp ''' + || TO_CHAR( d + INTERVAL '1 month', 'YYYY-MM-DD 00:00:00' ) + || ''' ) ) inherits ( histories );' + FROM generate_series( $1, $2, '1 month' ) AS d + LOOP + EXECUTE create_query; + END LOOP; -- LOOP END +END; -- FUNCTION END +$$ +language plpgsql; +-- +goose StatementEnd +``` + +## Go Migrations + +A sample Go migration looks like: + +```go +package main + +import ( + "fmt" + "github.com/jinzhu/gorm" +) + +func Up_20130106222315(txn *gorm.DB) { + fmt.Println("Hello from migration 20130106222315 Up!") +} + +func Down_20130106222315(txn *gorm.DB) { + fmt.Println("Hello from migration 20130106222315 Down!") +} + +``` + +`Up_20130106222315()` will be executed as part of a forward migration, and `Down_20130106222315()` will be executed as part of a rollback. + +The numeric portion of the function name (`20130106222315`) must be the leading portion of migration's filename, such as `20130106222315_descriptive_name.go`. `gorm-goose create` does this by default. + +A transaction is provided, rather than the DB instance directly, since gorm-goose also needs to record the schema version within the same transaction. Each migration should run as a single transaction to ensure DB integrity, so it's good practice anyway. + + +# Configuration + +gorm-goose expects you to maintain a folder (typically called "db"), which contains the following: + +* a `dbconf.yml` file that describes the database configurations you'd like to use +* a folder called "migrations" which contains `.sql` and/or `.go` scripts that implement your migrations + +You may use the `-path` option to specify an alternate location for the folder containing your config and migrations. + +A sample `dbconf.yml` looks like + +```yml +development: + driver: postgres + open: user=liam dbname=tester sslmode=disable +``` + +Here, `development` specifies the name of the environment, and the `driver` and `open` elements are passed directly to database/sql to access the specified database. + +You may include as many environments as you like, and you can use the `-env` command line option to specify which one to use. gorm-goose defaults to using an environment called `development`. + +gorm-goose will expand environment variables in the `open` element. For an example, see the Heroku section below. + +## Using goose with Heroku + +These instructions assume that you're using [Keith Rarick's Heroku Go buildpack](https://github.com/kr/heroku-buildpack-go). First, add a file to your project called (e.g.) `install_goose.go` to trigger building of the goose executable during deployment, with these contents: + +```go +// use build constraints to work around http://code.google.com/p/go/issues/detail?id=4210 +// +build heroku + +// note: need at least one blank line after build constraint +package main + +import _ "github.com/Altoros/gorm-goose/cmd/gorm-goose" +``` + +[Set up your Heroku database(s) as usual.](https://devcenter.heroku.com/articles/heroku-postgresql) + +Then make use of environment variable expansion in your `dbconf.yml`: + +```yml +production: + driver: postgres + open: $DATABASE_URL +``` + +To run gorm-goose in production, use `heroku run`: + + heroku run gorm-goose -env production up diff --git a/cmd/gorm-goose/cmd.go b/cmd/gorm-goose/cmd.go new file mode 100644 index 0000000..8a520d5 --- /dev/null +++ b/cmd/gorm-goose/cmd.go @@ -0,0 +1,27 @@ +package main + +import ( + "flag" +) + +// shamelessly snagged from the go tool +// each command gets its own set of args, +// defines its own entry point, and provides its own help +type Command struct { + Run func(cmd *Command, args ...string) + Flag flag.FlagSet + + Name string + Usage string + + Summary string + Help string +} + +func (c *Command) Exec(args []string) { + c.Flag.Usage = func() { + // helpFunc(c, c.Name) + } + c.Flag.Parse(args) + c.Run(c, c.Flag.Args()...) +} diff --git a/cmd/gorm-goose/cmd_create.go b/cmd/gorm-goose/cmd_create.go new file mode 100644 index 0000000..ee8cba9 --- /dev/null +++ b/cmd/gorm-goose/cmd_create.go @@ -0,0 +1,52 @@ +package main + +import ( + "fmt" + "log" + "os" + "path/filepath" + "time" + + "github.com/Altoros/gorm-goose/lib/goose" +) + +var createCmd = &Command{ + Name: "create", + Usage: "", + Summary: "Create the scaffolding for a new migration", + Help: `create extended help here...`, + Run: createRun, +} + +func createRun(cmd *Command, args ...string) { + + if len(args) < 1 { + log.Fatal("goose create: migration name required") + } + + migrationType := "go" // default to Go migrations + if len(args) >= 2 { + migrationType = args[1] + } + + conf, err := dbConfFromFlags() + if err != nil { + log.Fatal(err) + } + + if err = os.MkdirAll(conf.MigrationsDir, 0777); err != nil { + log.Fatal(err) + } + + n, err := goose.CreateMigration(args[0], migrationType, conf.MigrationsDir, time.Now()) + if err != nil { + log.Fatal(err) + } + + a, e := filepath.Abs(n) + if e != nil { + log.Fatal(e) + } + + fmt.Println("goose: created", a) +} diff --git a/cmd/gorm-goose/cmd_dbversion.go b/cmd/gorm-goose/cmd_dbversion.go new file mode 100644 index 0000000..b6437b6 --- /dev/null +++ b/cmd/gorm-goose/cmd_dbversion.go @@ -0,0 +1,30 @@ +package main + +import ( + "fmt" + "log" + + "github.com/Altoros/gorm-goose/lib/goose" +) + +var dbVersionCmd = &Command{ + Name: "dbversion", + Usage: "", + Summary: "Print the current version of the database", + Help: `dbversion extended help here...`, + Run: dbVersionRun, +} + +func dbVersionRun(cmd *Command, args ...string) { + conf, err := dbConfFromFlags() + if err != nil { + log.Fatal(err) + } + + current, err := goose.GetDBVersion(conf) + if err != nil { + log.Fatal(err) + } + + fmt.Printf("goose: dbversion %v\n", current) +} diff --git a/cmd/gorm-goose/cmd_down.go b/cmd/gorm-goose/cmd_down.go new file mode 100644 index 0000000..d32434b --- /dev/null +++ b/cmd/gorm-goose/cmd_down.go @@ -0,0 +1,37 @@ +package main + +import ( + "log" + + "github.com/Altoros/gorm-goose/lib/goose" +) + +var downCmd = &Command{ + Name: "down", + Usage: "", + Summary: "Roll back the version by 1", + Help: `down extended help here...`, + Run: downRun, +} + +func downRun(cmd *Command, args ...string) { + + conf, err := dbConfFromFlags() + if err != nil { + log.Fatal(err) + } + + current, err := goose.GetDBVersion(conf) + if err != nil { + log.Fatal(err) + } + + previous, err := goose.GetPreviousDBVersion(conf.MigrationsDir, current) + if err != nil { + log.Fatal(err) + } + + if err = goose.RunMigrations(conf, conf.MigrationsDir, previous); err != nil { + log.Fatal(err) + } +} diff --git a/cmd/gorm-goose/cmd_redo.go b/cmd/gorm-goose/cmd_redo.go new file mode 100644 index 0000000..eb78c8d --- /dev/null +++ b/cmd/gorm-goose/cmd_redo.go @@ -0,0 +1,40 @@ +package main + +import ( + "log" + + "github.com/Altoros/gorm-goose/lib/goose" +) + +var redoCmd = &Command{ + Name: "redo", + Usage: "", + Summary: "Re-run the latest migration", + Help: `redo extended help here...`, + Run: redoRun, +} + +func redoRun(cmd *Command, args ...string) { + conf, err := dbConfFromFlags() + if err != nil { + log.Fatal(err) + } + + current, err := goose.GetDBVersion(conf) + if err != nil { + log.Fatal(err) + } + + previous, err := goose.GetPreviousDBVersion(conf.MigrationsDir, current) + if err != nil { + log.Fatal(err) + } + + if err := goose.RunMigrations(conf, conf.MigrationsDir, previous); err != nil { + log.Fatal(err) + } + + if err := goose.RunMigrations(conf, conf.MigrationsDir, current); err != nil { + log.Fatal(err) + } +} diff --git a/cmd/gorm-goose/cmd_status.go b/cmd/gorm-goose/cmd_status.go new file mode 100644 index 0000000..6ac5933 --- /dev/null +++ b/cmd/gorm-goose/cmd_status.go @@ -0,0 +1,77 @@ +package main + +import ( + "fmt" + "log" + "path/filepath" + "time" + + "github.com/Altoros/gorm-goose/lib/goose" + "github.com/jinzhu/gorm" +) + +var statusCmd = &Command{ + Name: "status", + Usage: "", + Summary: "dump the migration status for the current DB", + Help: `status extended help here...`, + Run: statusRun, +} + +type StatusData struct { + Source string + Status string +} + +func statusRun(cmd *Command, args ...string) { + + conf, err := dbConfFromFlags() + if err != nil { + log.Fatal(err) + } + + // collect all migrations + min := int64(0) + max := int64((1 << 63) - 1) + migrations, e := goose.CollectMigrations(conf.MigrationsDir, min, max) + if e != nil { + log.Fatal(e) + } + + db, e := goose.OpenDBFromDBConf(conf) + if e != nil { + log.Fatal("couldn't open DB:", e) + } + defer db.Close() + + // must ensure that the version table exists if we're running on a pristine DB + if _, e := goose.EnsureDBVersion(conf, db); e != nil { + log.Fatal(e) + } + + fmt.Printf("goose: status for environment '%v'\n", conf.Env) + fmt.Println(" Applied At Migration") + fmt.Println(" =======================================") + for _, m := range migrations { + printMigrationStatus(db, m.Version, filepath.Base(m.Source)) + } +} + +func printMigrationStatus(db *gorm.DB, version int64, script string) { + row := goose.MigrationRecord{} + result := db.Where("version_id = ?", version).Order("t_stamp desc").First(&row) + + if result.Error != nil && !result.RecordNotFound() { + log.Fatal(result.Error) + } + + var appliedAt string + + if row.IsApplied { + appliedAt = row.TStamp.Format(time.ANSIC) + } else { + appliedAt = "Pending" + } + + fmt.Printf(" %-24s -- %v\n", appliedAt, script) +} diff --git a/cmd/gorm-goose/cmd_up.go b/cmd/gorm-goose/cmd_up.go new file mode 100644 index 0000000..5997dda --- /dev/null +++ b/cmd/gorm-goose/cmd_up.go @@ -0,0 +1,32 @@ +package main + +import ( + "log" + + "github.com/Altoros/gorm-goose/lib/goose" +) + +var upCmd = &Command{ + Name: "up", + Usage: "", + Summary: "Migrate the DB to the most recent version available", + Help: `up extended help here...`, + Run: upRun, +} + +func upRun(cmd *Command, args ...string) { + + conf, err := dbConfFromFlags() + if err != nil { + log.Fatal(err) + } + + target, err := goose.GetMostRecentDBVersion(conf.MigrationsDir) + if err != nil { + log.Fatal(err) + } + + if err := goose.RunMigrations(conf, conf.MigrationsDir, target); err != nil { + log.Fatal(err) + } +} diff --git a/cmd/gorm-goose/main.go b/cmd/gorm-goose/main.go new file mode 100644 index 0000000..478c620 --- /dev/null +++ b/cmd/gorm-goose/main.go @@ -0,0 +1,79 @@ +package main + +import ( + "flag" + "fmt" + "os" + "strings" + "text/template" + + "github.com/Altoros/gorm-goose/lib/goose" +) + +// global options. available to any subcommands. +var flagPath = flag.String("path", "db", "folder containing db info") +var flagEnv = flag.String("env", "development", "which DB environment to use") +var flagPgSchema = flag.String("pgschema", "", "which postgres-schema to migrate (default = none)") + +// helper to create a DBConf from the given flags +func dbConfFromFlags() (dbconf *goose.DBConf, err error) { + return goose.NewDBConf(*flagPath, *flagEnv, *flagPgSchema) +} + +var commands = []*Command{ + upCmd, + downCmd, + redoCmd, + statusCmd, + createCmd, + dbVersionCmd, +} + +func main() { + + flag.Usage = usage + flag.Parse() + + args := flag.Args() + if len(args) == 0 || args[0] == "-h" { + flag.Usage() + return + } + + var cmd *Command + name := args[0] + for _, c := range commands { + if strings.HasPrefix(c.Name, name) { + cmd = c + break + } + } + + if cmd == nil { + fmt.Printf("error: unknown command %q\n", name) + flag.Usage() + os.Exit(1) + } + + cmd.Exec(args[1:]) +} + +func usage() { + fmt.Print(usagePrefix) + flag.PrintDefaults() + usageTmpl.Execute(os.Stdout, commands) +} + +var usagePrefix = ` +gorm-goose is a database migration management system for Go projects. + +Usage: + gorm-goose [options] [subcommand options] + +Options: +` +var usageTmpl = template.Must(template.New("usage").Parse( + ` +Commands:{{range .}} + {{.Name | printf "%-10s"}} {{.Summary}}{{end}} +`)) diff --git a/db-sample/dbconf.yml b/db-sample/dbconf.yml new file mode 100644 index 0000000..3c986de --- /dev/null +++ b/db-sample/dbconf.yml @@ -0,0 +1,16 @@ + +test: + driver: mysql + open: user:password@/dbname + +development: + driver: postgres + open: host=myhost user=gorm dbname=gorm sslmode=disable password=mypassword + +production: + driver: sqlite3 + open: /tmp/gorm.db + +environment_variable_config: + driver: $DB_DRIVER + open: $DATABASE_URL diff --git a/db-sample/migrations/001_basics.sql b/db-sample/migrations/001_basics.sql new file mode 100644 index 0000000..2a5bb57 --- /dev/null +++ b/db-sample/migrations/001_basics.sql @@ -0,0 +1,11 @@ + +-- +goose Up +CREATE TABLE post ( + id int NOT NULL, + title text, + body text, + PRIMARY KEY(id) +); + +-- +goose Down +DROP TABLE post; diff --git a/db-sample/migrations/002_next.sql b/db-sample/migrations/002_next.sql new file mode 100644 index 0000000..9f9be33 --- /dev/null +++ b/db-sample/migrations/002_next.sql @@ -0,0 +1,12 @@ + +-- +goose Up +CREATE TABLE fancier_post ( + id int NOT NULL, + title text, + body text, + created_on timestamp without time zone, + PRIMARY KEY(id) +); + +-- +goose Down +DROP TABLE fancier_post; diff --git a/db-sample/migrations/20130106222315_and_again.go b/db-sample/migrations/20130106222315_and_again.go new file mode 100644 index 0000000..490fb41 --- /dev/null +++ b/db-sample/migrations/20130106222315_and_again.go @@ -0,0 +1,15 @@ +package main + +import ( + "fmt" + + "github.com/jinzhu/gorm" +) + +func Up_20130106222315(txn *gorm.DB) { + fmt.Println("Hello from migration 20130106222315 Up!") +} + +func Down_20130106222315(txn *gorm.DB) { + fmt.Println("Hello from migration 20130106222315 Down!") +} diff --git a/lib/goose/dbconf.go b/lib/goose/dbconf.go new file mode 100644 index 0000000..6531596 --- /dev/null +++ b/lib/goose/dbconf.go @@ -0,0 +1,116 @@ +package goose + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/jinzhu/gorm" + "github.com/kylelemons/go-gypsy/yaml" +) + +// DBDriver encapsulates the info needed to work with +// a specific database driver +type DBDriver struct { + Name string + OpenStr string + Import string +} + +type DBConf struct { + MigrationsDir string + Env string + Driver DBDriver + PgSchema string +} + +// NewDBConf extract configuration details from the given file +func NewDBConf(p, env string, pgschema string) (*DBConf, error) { + + cfgFile := filepath.Join(p, "dbconf.yml") + + f, err := yaml.ReadFile(cfgFile) + if err != nil { + return nil, err + } + + drv, err := f.Get(fmt.Sprintf("%s.driver", env)) + if err != nil { + return nil, err + } + drv = os.ExpandEnv(drv) + + open, err := f.Get(fmt.Sprintf("%s.open", env)) + if err != nil { + return nil, err + } + open = os.ExpandEnv(open) + + d := newDBDriver(drv, open) + + // allow the configuration to override the Import for this driver + if imprt, err := f.Get(fmt.Sprintf("%s.import", env)); err == nil { + d.Import = imprt + } + + if !d.IsValid() { + return nil, fmt.Errorf("Invalid DBConf: %v", d) + } + + return &DBConf{ + MigrationsDir: filepath.Join(p, "migrations"), + Env: env, + Driver: d, + PgSchema: pgschema, + }, nil +} + +// Create a new DBDriver and populate driver specific +// fields for drivers that we know about. +// Further customization may be done in NewDBConf +func newDBDriver(name, open string) DBDriver { + + d := DBDriver{ + Name: name, + OpenStr: open, + } + + switch name { + case "postgres": + d.Import = "github.com/jinzhu/gorm/dialects/postgres" + + case "mysql": + d.Import = "github.com/jinzhu/gorm/dialects/mysql" + d.OpenStr = d.OpenStr + "?charset=utf8&parseTime=True&loc=Local" + + case "sqlite3": + d.Import = "github.com/jinzhu/gorm/dialects/sqlite" + } + + return d +} + +// IsValid ensure we have enough info about this driver +func (drv *DBDriver) IsValid() bool { + return len(drv.Import) > 0 +} + +// OpenDBFromDBConf wraps database/sql.DB.Open() and configures +// the newly opened DB based on the given DBConf. +// +// Callers must Close() the returned DB. +func OpenDBFromDBConf(conf *DBConf) (*gorm.DB, error) { + db, err := gorm.Open(conf.Driver.Name, conf.Driver.OpenStr) + if err != nil { + return nil, err + } + + // if a postgres schema has been specified, apply it + if conf.Driver.Name == "postgres" && conf.PgSchema != "" { + if err := db.Exec("SET search_path TO " + conf.PgSchema).Error; err != nil { + return nil, err + } + } + + return db, nil +} diff --git a/lib/goose/dbconf_test.go b/lib/goose/dbconf_test.go new file mode 100644 index 0000000..db526b3 --- /dev/null +++ b/lib/goose/dbconf_test.go @@ -0,0 +1,69 @@ +package goose + +import ( + "os" + "testing" +) + +func TestBasics(t *testing.T) { + + dbconf, err := NewDBConf("../../db-sample", "test", "") + if err != nil { + t.Fatal(err) + } + + got := []string{dbconf.MigrationsDir, dbconf.Env, dbconf.Driver.Name, dbconf.Driver.OpenStr} + want := []string{"../../db-sample/migrations", "test", "postgres", "user=liam dbname=tester sslmode=disable"} + + for i, s := range got { + if s != want[i] { + t.Errorf("Unexpected DBConf value. got %v, want %v", s, want[i]) + } + } +} + +func TestImportOverride(t *testing.T) { + + dbconf, err := NewDBConf("../../db-sample", "customimport", "") + if err != nil { + t.Fatal(err) + } + + got := dbconf.Driver.Import + want := "github.com/custom/driver" + if got != want { + t.Errorf("bad custom import. got %v want %v", got, want) + } +} + +func TestDriverSetFromEnvironmentVariable(t *testing.T) { + + databaseUrlEnvVariableKey := "DB_DRIVER" + databaseUrlEnvVariableVal := "sqlite3" + databaseOpenStringKey := "DATABASE_URL" + databaseOpenStringVal := "db.db" + + os.Setenv(databaseUrlEnvVariableKey, databaseUrlEnvVariableVal) + os.Setenv(databaseOpenStringKey, databaseOpenStringVal) + + dbconf, err := NewDBConf("../../db-sample", "environment_variable_config", "") + if err != nil { + t.Fatal(err) + } + + got := dbconf.Driver.Name + want := "sqlite3" + + if got != want { + t.Errorf("Not able to read the driver type from environment variable."+ + "got %v want %v", got, want) + } + + gotOpenString := dbconf.Driver.OpenStr + wantOpenString := databaseOpenStringVal + + if gotOpenString != wantOpenString { + t.Errorf("Not able to read the open string from the environment."+ + "got %v want %v", gotOpenString, wantOpenString) + } +} diff --git a/lib/goose/migrate.go b/lib/goose/migrate.go new file mode 100644 index 0000000..483fcfc --- /dev/null +++ b/lib/goose/migrate.go @@ -0,0 +1,402 @@ +package goose + +import ( + "errors" + "fmt" + "log" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "text/template" + "time" + + "github.com/jinzhu/gorm" + _ "github.com/jinzhu/gorm/dialects/mysql" + _ "github.com/jinzhu/gorm/dialects/postgres" + _ "github.com/jinzhu/gorm/dialects/sqlite" +) + +var ( + ErrTableDoesNotExist = errors.New("table does not exist") + ErrNoPreviousVersion = errors.New("no previous version found") +) + +type MigrationRecord struct { + ID uint `gorm:"primary_key"` + VersionId int64 + TStamp time.Time `gorm:"default: now()"` + IsApplied bool // was this a result of up() or down() +} + +type Migration struct { + Version int64 + Next int64 // next version, or -1 if none + Previous int64 // previous version, -1 if none + Source string // path to .go or .sql script +} + +type migrationSorter []*Migration + +// helpers so we can use pkg sort +func (ms migrationSorter) Len() int { return len(ms) } +func (ms migrationSorter) Swap(i, j int) { ms[i], ms[j] = ms[j], ms[i] } +func (ms migrationSorter) Less(i, j int) bool { return ms[i].Version < ms[j].Version } + +func newMigration(v int64, src string) *Migration { + return &Migration{v, -1, -1, src} +} + +func RunMigrations(conf *DBConf, migrationsDir string, target int64) (err error) { + + db, err := OpenDBFromDBConf(conf) + if err != nil { + return err + } + defer db.Close() + + return RunMigrationsOnDb(conf, migrationsDir, target, db) +} + +// Runs migration on a specific database instance. +func RunMigrationsOnDb(conf *DBConf, migrationsDir string, target int64, db *gorm.DB) (err error) { + current, err := EnsureDBVersion(conf, db) + if err != nil { + return err + } + + migrations, err := CollectMigrations(migrationsDir, current, target) + if err != nil { + return err + } + + if len(migrations) == 0 { + fmt.Printf("goose: no migrations to run. current version: %d\n", current) + return nil + } + + ms := migrationSorter(migrations) + direction := current < target + ms.Sort(direction) + + fmt.Printf("goose: migrating db environment '%v', current version: %d, target: %d\n", + conf.Env, current, target) + + for _, m := range ms { + + switch filepath.Ext(m.Source) { + case ".go": + err = runGoMigration(conf, m.Source, m.Version, direction) + case ".sql": + err = runSQLMigration(conf, db, m.Source, m.Version, direction) + } + + if err != nil { + return errors.New(fmt.Sprintf("FAIL %v, quitting migration", err)) + } + + fmt.Println("OK ", filepath.Base(m.Source)) + } + + return nil +} + +// collect all the valid looking migration scripts in the +// migrations folder, and key them by version +func CollectMigrations(dirpath string, current, target int64) (m []*Migration, err error) { + + // extract the numeric component of each migration, + // filter out any uninteresting files, + // and ensure we only have one file per migration version. + filepath.Walk(dirpath, func(name string, info os.FileInfo, err error) error { + + if v, e := NumericComponent(name); e == nil { + + for _, g := range m { + if v == g.Version { + log.Fatalf("more than one file specifies the migration for version %d (%s and %s)", + v, g.Source, filepath.Join(dirpath, name)) + } + } + + if versionFilter(v, current, target) { + m = append(m, newMigration(v, name)) + } + } + + return nil + }) + + return m, nil +} + +func versionFilter(v, current, target int64) bool { + + if target > current { + return v > current && v <= target + } + + if target < current { + return v <= current && v > target + } + + return false +} + +func (ms migrationSorter) Sort(direction bool) { + + // sort ascending or descending by version + if direction { + sort.Sort(ms) + } else { + sort.Sort(sort.Reverse(ms)) + } + + // now that we're sorted in the appropriate direction, + // populate next and previous for each migration + for i, m := range ms { + prev := int64(-1) + if i > 0 { + prev = ms[i-1].Version + ms[i-1].Next = m.Version + } + ms[i].Previous = prev + } +} + +// look for migration scripts with names in the form: +// XXX_descriptivename.ext +// where XXX specifies the version number +// and ext specifies the type of migration +func NumericComponent(name string) (int64, error) { + + base := filepath.Base(name) + + if ext := filepath.Ext(base); ext != ".go" && ext != ".sql" { + return 0, errors.New("not a recognized migration file type") + } + + idx := strings.Index(base, "_") + if idx < 0 { + return 0, errors.New("no separator found") + } + + n, e := strconv.ParseInt(base[:idx], 10, 64) + if e == nil && n <= 0 { + return 0, errors.New("migration IDs must be greater than zero") + } + + return n, e +} + +// EnsureDBVersion retrieve the current version for this DB. +// Create and initialize the DB version table if it doesn't exist. +func EnsureDBVersion(conf *DBConf, db *gorm.DB) (int64, error) { + rows := []MigrationRecord{} + err := db.Order("id desc").Find(&rows).Error + + if err != nil { + return 0, createVersionTable(conf, db) + } + + // The most recent record for each migration specifies + // whether it has been applied or rolled back. + // The first version we find that has been applied is the current version. + + toSkip := make([]int64, 0) + + for _, row := range rows { + // have we already marked this version to be skipped? + skip := false + for _, v := range toSkip { + if v == row.VersionId { + skip = true + break + } + } + + if skip { + continue + } + + // if version has been applied we're done + if row.IsApplied { + return row.VersionId, nil + } + + // latest version of migration has not been applied. + toSkip = append(toSkip, row.VersionId) + } + + panic("failure in EnsureDBVersion()") +} + +// Create the goose_db_version table +// and insert the initial 0 value into it +func createVersionTable(conf *DBConf, db *gorm.DB) error { + txn := db.Begin() + if txn.Error != nil { + return txn.Error + } + + if err := txn.CreateTable(&MigrationRecord{}).Error; err != nil { + txn.Rollback() + return err + } + + record := MigrationRecord{VersionId: 0, IsApplied: true} + if err := txn.Create(&record).Error; err != nil { + txn.Rollback() + return err + } + + return txn.Commit().Error +} + +// wrapper for EnsureDBVersion for callers that don't already have +// their own DB instance +func GetDBVersion(conf *DBConf) (version int64, err error) { + + db, err := OpenDBFromDBConf(conf) + if err != nil { + return -1, err + } + defer db.Close() + + version, err = EnsureDBVersion(conf, db) + if err != nil { + return -1, err + } + + return version, nil +} + +func GetPreviousDBVersion(dirpath string, version int64) (previous int64, err error) { + + previous = -1 + sawGivenVersion := false + + filepath.Walk(dirpath, func(name string, info os.FileInfo, walkerr error) error { + + if !info.IsDir() { + if v, e := NumericComponent(name); e == nil { + if v > previous && v < version { + previous = v + } + if v == version { + sawGivenVersion = true + } + } + } + + return nil + }) + + if previous == -1 { + if sawGivenVersion { + // the given version is (likely) valid but we didn't find + // anything before it. + // 'previous' must reflect that no migrations have been applied. + previous = 0 + } else { + err = ErrNoPreviousVersion + } + } + + return +} + +// helper to identify the most recent possible version +// within a folder of migration scripts +func GetMostRecentDBVersion(dirpath string) (version int64, err error) { + + version = -1 + + filepath.Walk(dirpath, func(name string, info os.FileInfo, walkerr error) error { + if walkerr != nil { + return walkerr + } + + if !info.IsDir() { + if v, e := NumericComponent(name); e == nil { + if v > version { + version = v + } + } + } + + return nil + }) + + if version == -1 { + err = errors.New("no valid version found") + } + + return +} + +func CreateMigration(name, migrationType, dir string, t time.Time) (path string, err error) { + + if migrationType != "go" && migrationType != "sql" { + return "", errors.New("migration type must be 'go' or 'sql'") + } + + timestamp := t.Format("20060102150405") + filename := fmt.Sprintf("%v_%v.%v", timestamp, name, migrationType) + + fpath := filepath.Join(dir, filename) + + var tmpl *template.Template + if migrationType == "sql" { + tmpl = sqlMigrationTemplate + } else { + tmpl = goMigrationTemplate + } + + path, err = writeTemplateToFile(fpath, tmpl, timestamp) + + return +} + +// FinalizeMigration update the version table for the given migration, +// and finalize the transaction. +func FinalizeMigration(conf *DBConf, txn *gorm.DB, direction bool, v int64) error { + + // XXX: drop goose_db_version table on some minimum version number? + record := MigrationRecord{VersionId: v, IsApplied: direction} + if err := txn.Create(&record).Error; err != nil { + txn.Rollback() + return err + } + + return txn.Commit().Error +} + +var goMigrationTemplate = template.Must(template.New("goose.go-migration").Parse(` +package main + +import ( + "github.com/jinzhu/gorm" +) + +// Up is executed when this migration is applied +func Up_{{ . }}(txn *gorm.DB) { + +} + +// Down is executed when this migration is rolled back +func Down_{{ . }}(txn *gorm.DB) { + +} +`)) + +var sqlMigrationTemplate = template.Must(template.New("goose.sql-migration").Parse(` +-- +goose Up +-- SQL in section 'Up' is executed when this migration is applied + + +-- +goose Down +-- SQL section 'Down' is executed when this migration is rolled back + +`)) diff --git a/lib/goose/migrate_test.go b/lib/goose/migrate_test.go new file mode 100644 index 0000000..6dbffa3 --- /dev/null +++ b/lib/goose/migrate_test.go @@ -0,0 +1,71 @@ +package goose + +import ( + "testing" +) + +func TestMigrationMapSortUp(t *testing.T) { + + ms := migrationSorter{} + + // insert in any order + ms = append(ms, newMigration(20120000, "test")) + ms = append(ms, newMigration(20128000, "test")) + ms = append(ms, newMigration(20129000, "test")) + ms = append(ms, newMigration(20127000, "test")) + + ms.Sort(true) // sort Upwards + + sorted := []int64{20120000, 20127000, 20128000, 20129000} + + validateMigrationSort(t, ms, sorted) +} + +func TestMigrationMapSortDown(t *testing.T) { + + ms := migrationSorter{} + + // insert in any order + ms = append(ms, newMigration(20120000, "test")) + ms = append(ms, newMigration(20128000, "test")) + ms = append(ms, newMigration(20129000, "test")) + ms = append(ms, newMigration(20127000, "test")) + + ms.Sort(false) // sort Downwards + + sorted := []int64{20129000, 20128000, 20127000, 20120000} + + validateMigrationSort(t, ms, sorted) +} + +func validateMigrationSort(t *testing.T, ms migrationSorter, sorted []int64) { + + for i, m := range ms { + if sorted[i] != m.Version { + t.Error("incorrect sorted version") + } + + var next, prev int64 + + if i == 0 { + prev = -1 + next = ms[i+1].Version + } else if i == len(ms)-1 { + prev = ms[i-1].Version + next = -1 + } else { + prev = ms[i-1].Version + next = ms[i+1].Version + } + + if m.Next != next { + t.Errorf("mismatched Next. v: %v, got %v, wanted %v\n", m, m.Next, next) + } + + if m.Previous != prev { + t.Errorf("mismatched Previous v: %v, got %v, wanted %v\n", m, m.Previous, prev) + } + } + + t.Log(ms) +} diff --git a/lib/goose/migration_go.go b/lib/goose/migration_go.go new file mode 100644 index 0000000..318b5a4 --- /dev/null +++ b/lib/goose/migration_go.go @@ -0,0 +1,129 @@ +package goose + +import ( + "bytes" + "encoding/gob" + "fmt" + "io/ioutil" + "log" + "os" + "os/exec" + "path/filepath" + "text/template" +) + +type templateData struct { + Version int64 + Import string + Conf string // gob encoded DBConf + Direction bool + Func string +} + +// +// Run a .go migration. +// +// In order to do this, we copy a modified version of the +// original .go migration, and execute it via `go run` along +// with a main() of our own creation. +// +func runGoMigration(conf *DBConf, path string, version int64, direction bool) error { + + // everything gets written to a temp dir, and zapped afterwards + d, e := ioutil.TempDir("", "goose") + if e != nil { + log.Fatal(e) + } + defer os.RemoveAll(d) + + directionStr := "Down" + if direction { + directionStr = "Up" + } + + var bb bytes.Buffer + if err := gob.NewEncoder(&bb).Encode(conf); err != nil { + return err + } + + // XXX: there must be a better way of making this byte array + // available to the generated code... + // but for now, print an array literal of the gob bytes + var sb bytes.Buffer + sb.WriteString("[]byte{ ") + for _, b := range bb.Bytes() { + sb.WriteString(fmt.Sprintf("0x%02x, ", b)) + } + sb.WriteString("}") + + td := &templateData{ + Version: version, + Import: conf.Driver.Import, + Conf: sb.String(), + Direction: direction, + Func: fmt.Sprintf("%v_%v", directionStr, version), + } + main, e := writeTemplateToFile(filepath.Join(d, "goose_main.go"), goMigrationDriverTemplate, td) + if e != nil { + log.Fatal(e) + } + + outpath := filepath.Join(d, filepath.Base(path)) + if _, e = copyFile(outpath, path); e != nil { + log.Fatal(e) + } + + cmd := exec.Command("go", "run", main, outpath) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if e = cmd.Run(); e != nil { + log.Fatal("`go run` failed: ", e) + } + + return nil +} + +// +// template for the main entry point to a go-based migration. +// this gets linked against the substituted versions of the user-supplied +// scripts in order to execute a migration via `go run` +// +var goMigrationDriverTemplate = template.Must(template.New("goose.go-driver").Parse(` +package main + +import ( + "log" + "bytes" + "encoding/gob" + + _ "{{.Import}}" + "github.com/Altoros/gorm-goose/lib/goose" +) + +func main() { + + var conf goose.DBConf + buf := bytes.NewBuffer({{ .Conf }}) + if err := gob.NewDecoder(buf).Decode(&conf); err != nil { + log.Fatal("gob.Decode - ", err) + } + + db, err := goose.OpenDBFromDBConf(&conf) + if err != nil { + log.Fatal("failed to open DB:", err) + } + defer db.Close() + + txn := db.Begin() + if txn.Error != nil { + log.Fatal("db.Begin:", txn.Error) + } + + {{ .Func }}(txn) + + err = goose.FinalizeMigration(&conf, txn, {{ .Direction }}, {{ .Version }}) + if err != nil { + log.Fatal("Commit() failed:", err) + } +} +`)) diff --git a/lib/goose/migration_sql.go b/lib/goose/migration_sql.go new file mode 100644 index 0000000..7a56891 --- /dev/null +++ b/lib/goose/migration_sql.go @@ -0,0 +1,168 @@ +package goose + +import ( + "bufio" + "bytes" + "io" + "log" + "os" + "path/filepath" + "strings" + + "github.com/jinzhu/gorm" +) + +const sqlCmdPrefix = "-- +goose " + +// Checks the line to see if the line has a statement-ending semicolon +// or if the line contains a double-dash comment. +func endsWithSemicolon(line string) bool { + + prev := "" + scanner := bufio.NewScanner(strings.NewReader(line)) + scanner.Split(bufio.ScanWords) + + for scanner.Scan() { + word := scanner.Text() + if strings.HasPrefix(word, "--") { + break + } + prev = word + } + + return strings.HasSuffix(prev, ";") +} + +// Split the given sql script into individual statements. +// +// The base case is to simply split on semicolons, as these +// naturally terminate a statement. +// +// However, more complex cases like pl/pgsql can have semicolons +// within a statement. For these cases, we provide the explicit annotations +// 'StatementBegin' and 'StatementEnd' to allow the script to +// tell us to ignore semicolons. +func splitSQLStatements(r io.Reader, direction bool) (stmts []string) { + + var buf bytes.Buffer + scanner := bufio.NewScanner(r) + + // track the count of each section + // so we can diagnose scripts with no annotations + upSections := 0 + downSections := 0 + + statementEnded := false + ignoreSemicolons := false + directionIsActive := false + + for scanner.Scan() { + + line := scanner.Text() + + // handle any goose-specific commands + if strings.HasPrefix(line, sqlCmdPrefix) { + cmd := strings.TrimSpace(line[len(sqlCmdPrefix):]) + switch cmd { + case "Up": + directionIsActive = (direction == true) + upSections++ + break + + case "Down": + directionIsActive = (direction == false) + downSections++ + break + + case "StatementBegin": + if directionIsActive { + ignoreSemicolons = true + } + break + + case "StatementEnd": + if directionIsActive { + statementEnded = (ignoreSemicolons == true) + ignoreSemicolons = false + } + break + } + } + + if !directionIsActive { + continue + } + + if _, err := buf.WriteString(line + "\n"); err != nil { + log.Fatalf("io err: %v", err) + } + + // Wrap up the two supported cases: 1) basic with semicolon; 2) psql statement + // Lines that end with semicolon that are in a statement block + // do not conclude statement. + if (!ignoreSemicolons && endsWithSemicolon(line)) || statementEnded { + statementEnded = false + stmts = append(stmts, buf.String()) + buf.Reset() + } + } + + if err := scanner.Err(); err != nil { + log.Fatalf("scanning migration: %v", err) + } + + // diagnose likely migration script errors + if ignoreSemicolons { + log.Println("WARNING: saw '-- +goose StatementBegin' with no matching '-- +goose StatementEnd'") + } + + if bufferRemaining := strings.TrimSpace(buf.String()); len(bufferRemaining) > 0 { + log.Printf("WARNING: Unexpected unfinished SQL query: %s. Missing a semicolon?\n", bufferRemaining) + } + + if upSections == 0 && downSections == 0 { + log.Fatal("ERROR: no Up/Down annotations found, so no statements were executed.") + } + + return +} + +// Run a migration specified in raw SQL. +// +// Sections of the script can be annotated with a special comment, +// starting with "-- +goose" to specify whether the section should +// be applied during an Up or Down migration +// +// All statements following an Up or Down directive are grouped together +// until another direction directive is found. +func runSQLMigration(conf *DBConf, db *gorm.DB, scriptFile string, v int64, direction bool) error { + + txn := db.Begin() + if txn.Error != nil { + log.Fatal("db.Begin:", txn.Error) + } + + f, err := os.Open(scriptFile) + if err != nil { + log.Fatal(err) + } + + // find each statement, checking annotations for up/down direction + // and execute each of them in the current transaction. + // Commits the transaction if successfully applied each statement and + // records the version into the version table or returns an error and + // rolls back the transaction. + for _, query := range splitSQLStatements(f, direction) { + if err = txn.Exec(query).Error; err != nil { + txn.Rollback() + log.Fatalf("FAIL %s (%v), quitting migration.", filepath.Base(scriptFile), err) + return err + } + } + + if err = FinalizeMigration(conf, txn, direction, v); err != nil { + log.Fatalf("error finalizing migration %s, quitting. (%v)", filepath.Base(scriptFile), err) + } + + return nil +} diff --git a/lib/goose/migration_sql_test.go b/lib/goose/migration_sql_test.go new file mode 100644 index 0000000..5852960 --- /dev/null +++ b/lib/goose/migration_sql_test.go @@ -0,0 +1,147 @@ +package goose + +import ( + "strings" + "testing" +) + +func TestSemicolons(t *testing.T) { + + type testData struct { + line string + result bool + } + + tests := []testData{ + { + line: "END;", + result: true, + }, + { + line: "END; -- comment", + result: true, + }, + { + line: "END ; -- comment", + result: true, + }, + { + line: "END -- comment", + result: false, + }, + { + line: "END -- comment ;", + result: false, + }, + { + line: "END \" ; \" -- comment", + result: false, + }, + } + + for _, test := range tests { + r := endsWithSemicolon(test.line) + if r != test.result { + t.Errorf("incorrect semicolon. got %v, want %v", r, test.result) + } + } +} + +func TestSplitStatements(t *testing.T) { + + type testData struct { + sql string + direction bool + count int + } + + tests := []testData{ + { + sql: functxt, + direction: true, + count: 2, + }, + { + sql: functxt, + direction: false, + count: 2, + }, + { + sql: multitxt, + direction: true, + count: 2, + }, + { + sql: multitxt, + direction: false, + count: 2, + }, + } + + for _, test := range tests { + stmts := splitSQLStatements(strings.NewReader(test.sql), test.direction) + if len(stmts) != test.count { + t.Errorf("incorrect number of stmts. got %v, want %v", len(stmts), test.count) + } + } +} + +var functxt = `-- +goose Up +CREATE TABLE IF NOT EXISTS histories ( + id BIGSERIAL PRIMARY KEY, + current_value varchar(2000) NOT NULL, + created_at timestamp with time zone NOT NULL +); + +-- +goose StatementBegin +CREATE OR REPLACE FUNCTION histories_partition_creation( DATE, DATE ) +returns void AS $$ +DECLARE + create_query text; +BEGIN + FOR create_query IN SELECT + 'CREATE TABLE IF NOT EXISTS histories_' + || TO_CHAR( d, 'YYYY_MM' ) + || ' ( CHECK( created_at >= timestamp ''' + || TO_CHAR( d, 'YYYY-MM-DD 00:00:00' ) + || ''' AND created_at < timestamp ''' + || TO_CHAR( d + INTERVAL '1 month', 'YYYY-MM-DD 00:00:00' ) + || ''' ) ) inherits ( histories );' + FROM generate_series( $1, $2, '1 month' ) AS d + LOOP + EXECUTE create_query; + END LOOP; -- LOOP END +END; -- FUNCTION END +$$ +language plpgsql; +-- +goose StatementEnd + +-- +goose Down +drop function histories_partition_creation(DATE, DATE); +drop TABLE histories; +` + +// test multiple up/down transitions in a single script +var multitxt = `-- +goose Up +CREATE TABLE post ( + id int NOT NULL, + title text, + body text, + PRIMARY KEY(id) +); + +-- +goose Down +DROP TABLE post; + +-- +goose Up +CREATE TABLE fancier_post ( + id int NOT NULL, + title text, + body text, + created_on timestamp without time zone, + PRIMARY KEY(id) +); + +-- +goose Down +DROP TABLE fancier_post; +` diff --git a/lib/goose/util.go b/lib/goose/util.go new file mode 100644 index 0000000..6296923 --- /dev/null +++ b/lib/goose/util.go @@ -0,0 +1,40 @@ +package goose + +import ( + "io" + "os" + "text/template" +) + +// common routines + +func writeTemplateToFile(path string, t *template.Template, data interface{}) (string, error) { + f, e := os.Create(path) + if e != nil { + return "", e + } + defer f.Close() + + e = t.Execute(f, data) + if e != nil { + return "", e + } + + return f.Name(), nil +} + +func copyFile(dst, src string) (int64, error) { + sf, err := os.Open(src) + if err != nil { + return 0, err + } + defer sf.Close() + + df, err := os.Create(dst) + if err != nil { + return 0, err + } + defer df.Close() + + return io.Copy(df, sf) +}