go-migration/gorm-goose
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

add library itself

Browse Source
This commit is contained in:
Vladimir Sagan 2016-09-09 15:12:30 +03:00
parent 854f436599
commit b6684a18ac
22 changed files with 1798 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
.gitignore vendored Normal file
View File

229
README.md
View File

@ -1 +1,228 @@
gorm-goose
# gorm-goose
This is a fork of [https://bitbucket.org/liamstask/goose](https://bitbucket.org/liamstask/goose) for [gorm](https://github.com/jinzhu/gorm).
gorm-goose is a database migration tool for [gorm](https://github.com/jinzhu/gorm).
Currently, available drivers are: "postgres", "mysql", or "sqlite3".
You can manage your database's evolution by creating incremental SQL or Go scripts.
# Install
$ go get github.com/Altoros/gorm-goose/cmd/gorm-goose
This will install the `gorm-goose` binary to your `$GOPATH/bin` directory.
You can also build gorm-goose into your own applications by importing `github.com/Altoros/gorm-goose/lib/goose`.
# Usage
gorm-goose provides several commands to help manage your database schema.
## create
Create a new Go migration.
$ gorm-goose create AddSomeColumns
$ goose: created db/migrations/20130106093224_AddSomeColumns.go
Edit the newly created script to define the behavior of your migration.
You can also create an SQL migration:
$ gorm-goose create AddSomeColumns sql
$ goose: created db/migrations/20130106093224_AddSomeColumns.sql
## up
Apply all available migrations.
$ gorm-goose up
$ goose: migrating db environment 'development', current version: 0, target: 3
$ OK 001_basics.sql
$ OK 002_next.sql
$ OK 003_and_again.go
### option: pgschema
Use the `pgschema` flag with the `up` command specify a postgres schema.
$ gorm-goose -pgschema=my_schema_name up
$ goose: migrating db environment 'development', current version: 0, target: 3
$ OK 001_basics.sql
$ OK 002_next.sql
$ OK 003_and_again.go
## down
Roll back a single migration from the current version.
$ gorm-goose down
$ goose: migrating db environment 'development', current version: 3, target: 2
$ OK 003_and_again.go
## redo
Roll back the most recently applied migration, then run it again.
$ gorm-goose redo
$ goose: migrating db environment 'development', current version: 3, target: 2
$ OK 003_and_again.go
$ goose: migrating db environment 'development', current version: 2, target: 3
$ OK 003_and_again.go
## status
Print the status of all migrations:
$ gorm-goose status
$ goose: status for environment 'development'
$ Applied At Migration
$ =======================================
$ Sun Jan 6 11:25:03 2013 -- 001_basics.sql
$ Sun Jan 6 11:25:03 2013 -- 002_next.sql
$ Pending -- 003_and_again.go
## dbversion
Print the current version of the database:
$ gorm-goose dbversion
$ goose: dbversion 002
`gorm-goose -h` provides more detailed info on each command.
# Migrations
gorm-goose supports migrations written in SQL or in Go - see the `gorm-goose create` command above for details on how to generate them.
## SQL Migrations
A sample SQL migration looks like:
```sql
-- +goose Up
CREATE TABLE post (
id int NOT NULL,
title text,
body text,
PRIMARY KEY(id)
);
-- +goose Down
DROP TABLE post;
```
Notice the annotations in the comments. Any statements following `-- +goose Up` will be executed as part of a forward migration, and any statements following `-- +goose Down` will be executed as part of a rollback.
By default, SQL statements are delimited by semicolons - in fact, query statements must end with a semicolon to be properly recognized by goose.
More complex statements (PL/pgSQL) that have semicolons within them must be annotated with `-- +goose StatementBegin` and `-- +goose StatementEnd` to be properly recognized. For example:
```sql
-- +goose Up
-- +goose StatementBegin
CREATE OR REPLACE FUNCTION histories_partition_creation( DATE, DATE )
returns void AS $$
DECLARE
create_query text;
BEGIN
FOR create_query IN SELECT
'CREATE TABLE IF NOT EXISTS histories_'
|| TO_CHAR( d, 'YYYY_MM' )
|| ' ( CHECK( created_at >= timestamp '''
|| TO_CHAR( d, 'YYYY-MM-DD 00:00:00' )
|| ''' AND created_at < timestamp '''
|| TO_CHAR( d + INTERVAL '1 month', 'YYYY-MM-DD 00:00:00' )
|| ''' ) ) inherits ( histories );'
FROM generate_series( $1, $2, '1 month' ) AS d
LOOP
EXECUTE create_query;
END LOOP; -- LOOP END
END; -- FUNCTION END
$$
language plpgsql;
-- +goose StatementEnd
```
## Go Migrations
A sample Go migration looks like:
```go
package main
import (
"fmt"
"github.com/jinzhu/gorm"
)
func Up_20130106222315(txn *gorm.DB) {
fmt.Println("Hello from migration 20130106222315 Up!")
}
func Down_20130106222315(txn *gorm.DB) {
fmt.Println("Hello from migration 20130106222315 Down!")
}
```
`Up_20130106222315()` will be executed as part of a forward migration, and `Down_20130106222315()` will be executed as part of a rollback.
The numeric portion of the function name (`20130106222315`) must be the leading portion of migration's filename, such as `20130106222315_descriptive_name.go`. `gorm-goose create` does this by default.
A transaction is provided, rather than the DB instance directly, since gorm-goose also needs to record the schema version within the same transaction. Each migration should run as a single transaction to ensure DB integrity, so it's good practice anyway.
# Configuration
gorm-goose expects you to maintain a folder (typically called "db"), which contains the following:
* a `dbconf.yml` file that describes the database configurations you'd like to use
* a folder called "migrations" which contains `.sql` and/or `.go` scripts that implement your migrations
You may use the `-path` option to specify an alternate location for the folder containing your config and migrations.
A sample `dbconf.yml` looks like
```yml
development:
driver: postgres
open: user=liam dbname=tester sslmode=disable
```
Here, `development` specifies the name of the environment, and the `driver` and `open` elements are passed directly to database/sql to access the specified database.
You may include as many environments as you like, and you can use the `-env` command line option to specify which one to use. gorm-goose defaults to using an environment called `development`.
gorm-goose will expand environment variables in the `open` element. For an example, see the Heroku section below.
## Using goose with Heroku
These instructions assume that you're using [Keith Rarick's Heroku Go buildpack](https://github.com/kr/heroku-buildpack-go). First, add a file to your project called (e.g.) `install_goose.go` to trigger building of the goose executable during deployment, with these contents:
```go
// use build constraints to work around http://code.google.com/p/go/issues/detail?id=4210
// +build heroku
// note: need at least one blank line after build constraint
package main
import _ "github.com/Altoros/gorm-goose/cmd/gorm-goose"
```
[Set up your Heroku database(s) as usual.](https://devcenter.heroku.com/articles/heroku-postgresql)
Then make use of environment variable expansion in your `dbconf.yml`:
```yml
production:
driver: postgres
open: $DATABASE_URL
```
To run gorm-goose in production, use `heroku run`:
heroku run gorm-goose -env production up

27
cmd/gorm-goose/cmd.go Normal file
View File

@ -0,0 +1,27 @@
package main
import (
"flag"
)
// shamelessly snagged from the go tool
// each command gets its own set of args,
// defines its own entry point, and provides its own help
type Command struct {
Run func(cmd *Command, args ...string)
Flag flag.FlagSet
Name string
Usage string
Summary string
Help string
}
func (c *Command) Exec(args []string) {
c.Flag.Usage = func() {
// helpFunc(c, c.Name)
}
c.Flag.Parse(args)
c.Run(c, c.Flag.Args()...)
}

52
cmd/gorm-goose/cmd_create.go Normal file
View File

@ -0,0 +1,52 @@
package main
import (
"fmt"
"log"
"os"
"path/filepath"
"time"
"github.com/Altoros/gorm-goose/lib/goose"
)
var createCmd = &Command{
Name: "create",
Usage: "",
Summary: "Create the scaffolding for a new migration",
Help: `create extended help here...`,
Run: createRun,
}
func createRun(cmd *Command, args ...string) {
if len(args) < 1 {
log.Fatal("goose create: migration name required")
}
migrationType := "go" // default to Go migrations
if len(args) >= 2 {
migrationType = args[1]
}
conf, err := dbConfFromFlags()
if err != nil {
log.Fatal(err)
}
if err = os.MkdirAll(conf.MigrationsDir, 0777); err != nil {
log.Fatal(err)
}
n, err := goose.CreateMigration(args[0], migrationType, conf.MigrationsDir, time.Now())
if err != nil {
log.Fatal(err)
}
a, e := filepath.Abs(n)
if e != nil {
log.Fatal(e)
}
fmt.Println("goose: created", a)
}

30
cmd/gorm-goose/cmd_dbversion.go Normal file
View File

@ -0,0 +1,30 @@
package main
import (
"fmt"
"log"
"github.com/Altoros/gorm-goose/lib/goose"
)
var dbVersionCmd = &Command{
Name: "dbversion",
Usage: "",
Summary: "Print the current version of the database",
Help: `dbversion extended help here...`,
Run: dbVersionRun,
}
func dbVersionRun(cmd *Command, args ...string) {
conf, err := dbConfFromFlags()
if err != nil {
log.Fatal(err)
}
current, err := goose.GetDBVersion(conf)
if err != nil {
log.Fatal(err)
}
fmt.Printf("goose: dbversion %v\n", current)
}

37
cmd/gorm-goose/cmd_down.go Normal file
View File

@ -0,0 +1,37 @@
package main
import (
"log"
"github.com/Altoros/gorm-goose/lib/goose"
)
var downCmd = &Command{
Name: "down",
Usage: "",
Summary: "Roll back the version by 1",
Help: `down extended help here...`,
Run: downRun,
}
func downRun(cmd *Command, args ...string) {
conf, err := dbConfFromFlags()
if err != nil {
log.Fatal(err)
}
current, err := goose.GetDBVersion(conf)
if err != nil {
log.Fatal(err)
}
previous, err := goose.GetPreviousDBVersion(conf.MigrationsDir, current)
if err != nil {
log.Fatal(err)
}
if err = goose.RunMigrations(conf, conf.MigrationsDir, previous); err != nil {
log.Fatal(err)
}
}

40
cmd/gorm-goose/cmd_redo.go Normal file
View File

@ -0,0 +1,40 @@
package main
import (
"log"
"github.com/Altoros/gorm-goose/lib/goose"
)
var redoCmd = &Command{
Name: "redo",
Usage: "",
Summary: "Re-run the latest migration",
Help: `redo extended help here...`,
Run: redoRun,
}
func redoRun(cmd *Command, args ...string) {
conf, err := dbConfFromFlags()
if err != nil {
log.Fatal(err)
}
current, err := goose.GetDBVersion(conf)
if err != nil {
log.Fatal(err)
}
previous, err := goose.GetPreviousDBVersion(conf.MigrationsDir, current)
if err != nil {
log.Fatal(err)
}
if err := goose.RunMigrations(conf, conf.MigrationsDir, previous); err != nil {
log.Fatal(err)
}
if err := goose.RunMigrations(conf, conf.MigrationsDir, current); err != nil {
log.Fatal(err)
}
}

77
cmd/gorm-goose/cmd_status.go Normal file
View File

@ -0,0 +1,77 @@
package main
import (
"fmt"
"log"
"path/filepath"
"time"
"github.com/Altoros/gorm-goose/lib/goose"
"github.com/jinzhu/gorm"
)
var statusCmd = &Command{
Name: "status",
Usage: "",
Summary: "dump the migration status for the current DB",
Help: `status extended help here...`,
Run: statusRun,
}
type StatusData struct {
Source string
Status string
}
func statusRun(cmd *Command, args ...string) {
conf, err := dbConfFromFlags()
if err != nil {
log.Fatal(err)
}
// collect all migrations
min := int64(0)
max := int64((1 << 63) - 1)
migrations, e := goose.CollectMigrations(conf.MigrationsDir, min, max)
if e != nil {
log.Fatal(e)
}
db, e := goose.OpenDBFromDBConf(conf)
if e != nil {
log.Fatal("couldn't open DB:", e)
}
defer db.Close()
// must ensure that the version table exists if we're running on a pristine DB
if _, e := goose.EnsureDBVersion(conf, db); e != nil {
log.Fatal(e)
}
fmt.Printf("goose: status for environment '%v'\n", conf.Env)
fmt.Println(" Applied At Migration")
fmt.Println(" =======================================")
for _, m := range migrations {
printMigrationStatus(db, m.Version, filepath.Base(m.Source))
}
}
func printMigrationStatus(db *gorm.DB, version int64, script string) {
row := goose.MigrationRecord{}
result := db.Where("version_id = ?", version).Order("t_stamp desc").First(&row)
if result.Error != nil && !result.RecordNotFound() {
log.Fatal(result.Error)
}
var appliedAt string
if row.IsApplied {
appliedAt = row.TStamp.Format(time.ANSIC)
} else {
appliedAt = "Pending"
}
fmt.Printf(" %-24s -- %v\n", appliedAt, script)
}

32
cmd/gorm-goose/cmd_up.go Normal file
View File

@ -0,0 +1,32 @@
package main
import (
"log"
"github.com/Altoros/gorm-goose/lib/goose"
)
var upCmd = &Command{
Name: "up",
Usage: "",
Summary: "Migrate the DB to the most recent version available",
Help: `up extended help here...`,
Run: upRun,
}
func upRun(cmd *Command, args ...string) {
conf, err := dbConfFromFlags()
if err != nil {
log.Fatal(err)
}
target, err := goose.GetMostRecentDBVersion(conf.MigrationsDir)
if err != nil {
log.Fatal(err)
}
if err := goose.RunMigrations(conf, conf.MigrationsDir, target); err != nil {
log.Fatal(err)
}
}

79
cmd/gorm-goose/main.go Normal file
View File

@ -0,0 +1,79 @@
package main
import (
"flag"
"fmt"
"os"
"strings"
"text/template"
"github.com/Altoros/gorm-goose/lib/goose"
)
// global options. available to any subcommands.
var flagPath = flag.String("path", "db", "folder containing db info")
var flagEnv = flag.String("env", "development", "which DB environment to use")
var flagPgSchema = flag.String("pgschema", "", "which postgres-schema to migrate (default = none)")
// helper to create a DBConf from the given flags
func dbConfFromFlags() (dbconf *goose.DBConf, err error) {
return goose.NewDBConf(*flagPath, *flagEnv, *flagPgSchema)
}
var commands = []*Command{
upCmd,
downCmd,
redoCmd,
statusCmd,
createCmd,
dbVersionCmd,
}
func main() {
flag.Usage = usage
flag.Parse()
args := flag.Args()
if len(args) == 0 || args[0] == "-h" {
flag.Usage()
return
}
var cmd *Command
name := args[0]
for _, c := range commands {
if strings.HasPrefix(c.Name, name) {
cmd = c
break
}
}
if cmd == nil {
fmt.Printf("error: unknown command %q\n", name)
flag.Usage()
os.Exit(1)
}
cmd.Exec(args[1:])
}
func usage() {
fmt.Print(usagePrefix)
flag.PrintDefaults()
usageTmpl.Execute(os.Stdout, commands)
}
var usagePrefix = `
gorm-goose is a database migration management system for Go projects.
Usage:
gorm-goose [options] <subcommand> [subcommand options]
Options:
`
var usageTmpl = template.Must(template.New("usage").Parse(
`
Commands:{{range .}}
{{.Name | printf "%-10s"}} {{.Summary}}{{end}}
`))

16
db-sample/dbconf.yml Normal file
View File

@ -0,0 +1,16 @@
test:
driver: mysql
open: user:password@/dbname
development:
driver: postgres
open: host=myhost user=gorm dbname=gorm sslmode=disable password=mypassword
production:
driver: sqlite3
open: /tmp/gorm.db
environment_variable_config:
driver: $DB_DRIVER
open: $DATABASE_URL

11
db-sample/migrations/001_basics.sql Normal file
View File

@ -0,0 +1,11 @@
-- +goose Up
CREATE TABLE post (
id int NOT NULL,
title text,
body text,
PRIMARY KEY(id)
);
-- +goose Down
DROP TABLE post;

12
db-sample/migrations/002_next.sql Normal file
View File

@ -0,0 +1,12 @@
-- +goose Up
CREATE TABLE fancier_post (
id int NOT NULL,
title text,
body text,
created_on timestamp without time zone,
PRIMARY KEY(id)
);
-- +goose Down
DROP TABLE fancier_post;

15
db-sample/migrations/20130106222315_and_again.go Normal file
View File

@ -0,0 +1,15 @@
package main
import (
"fmt"
"github.com/jinzhu/gorm"
)
func Up_20130106222315(txn *gorm.DB) {
fmt.Println("Hello from migration 20130106222315 Up!")
}
func Down_20130106222315(txn *gorm.DB) {
fmt.Println("Hello from migration 20130106222315 Down!")
}

116
lib/goose/dbconf.go Normal file
View File

@ -0,0 +1,116 @@
package goose
import (
"fmt"
"os"
"path/filepath"
"github.com/jinzhu/gorm"
"github.com/kylelemons/go-gypsy/yaml"
)
// DBDriver encapsulates the info needed to work with
// a specific database driver
type DBDriver struct {
Name string
OpenStr string
Import string
}
type DBConf struct {
MigrationsDir string
Env string
Driver DBDriver
PgSchema string
}
// NewDBConf extract configuration details from the given file
func NewDBConf(p, env string, pgschema string) (*DBConf, error) {
cfgFile := filepath.Join(p, "dbconf.yml")
f, err := yaml.ReadFile(cfgFile)
if err != nil {
return nil, err
}
drv, err := f.Get(fmt.Sprintf("%s.driver", env))
if err != nil {
return nil, err
}
drv = os.ExpandEnv(drv)
open, err := f.Get(fmt.Sprintf("%s.open", env))
if err != nil {
return nil, err
}
open = os.ExpandEnv(open)
d := newDBDriver(drv, open)
// allow the configuration to override the Import for this driver
if imprt, err := f.Get(fmt.Sprintf("%s.import", env)); err == nil {
d.Import = imprt
}
if !d.IsValid() {
return nil, fmt.Errorf("Invalid DBConf: %v", d)
}
return &DBConf{
MigrationsDir: filepath.Join(p, "migrations"),
Env: env,
Driver: d,
PgSchema: pgschema,
}, nil
}
// Create a new DBDriver and populate driver specific
// fields for drivers that we know about.
// Further customization may be done in NewDBConf
func newDBDriver(name, open string) DBDriver {
d := DBDriver{
Name: name,
OpenStr: open,
}
switch name {
case "postgres":
d.Import = "github.com/jinzhu/gorm/dialects/postgres"
case "mysql":
d.Import = "github.com/jinzhu/gorm/dialects/mysql"
d.OpenStr = d.OpenStr + "?charset=utf8&parseTime=True&loc=Local"
case "sqlite3":
d.Import = "github.com/jinzhu/gorm/dialects/sqlite"
}
return d
}
// IsValid ensure we have enough info about this driver
func (drv *DBDriver) IsValid() bool {
return len(drv.Import) > 0
}
// OpenDBFromDBConf wraps database/sql.DB.Open() and configures
// the newly opened DB based on the given DBConf.
//
// Callers must Close() the returned DB.
func OpenDBFromDBConf(conf *DBConf) (*gorm.DB, error) {
db, err := gorm.Open(conf.Driver.Name, conf.Driver.OpenStr)
if err != nil {
return nil, err
}
// if a postgres schema has been specified, apply it
if conf.Driver.Name == "postgres" && conf.PgSchema != "" {
if err := db.Exec("SET search_path TO " + conf.PgSchema).Error; err != nil {
return nil, err
}
}
return db, nil
}

69
lib/goose/dbconf_test.go Normal file
View File

@ -0,0 +1,69 @@
package goose
import (
"os"
"testing"
)
func TestBasics(t *testing.T) {
dbconf, err := NewDBConf("../../db-sample", "test", "")
if err != nil {
t.Fatal(err)
}
got := []string{dbconf.MigrationsDir, dbconf.Env, dbconf.Driver.Name, dbconf.Driver.OpenStr}
want := []string{"../../db-sample/migrations", "test", "postgres", "user=liam dbname=tester sslmode=disable"}
for i, s := range got {
if s != want[i] {
t.Errorf("Unexpected DBConf value. got %v, want %v", s, want[i])
}
}
}
func TestImportOverride(t *testing.T) {
dbconf, err := NewDBConf("../../db-sample", "customimport", "")
if err != nil {
t.Fatal(err)
}
got := dbconf.Driver.Import
want := "github.com/custom/driver"
if got != want {
t.Errorf("bad custom import. got %v want %v", got, want)
}
}
func TestDriverSetFromEnvironmentVariable(t *testing.T) {
databaseUrlEnvVariableKey := "DB_DRIVER"
databaseUrlEnvVariableVal := "sqlite3"
databaseOpenStringKey := "DATABASE_URL"
databaseOpenStringVal := "db.db"
os.Setenv(databaseUrlEnvVariableKey, databaseUrlEnvVariableVal)
os.Setenv(databaseOpenStringKey, databaseOpenStringVal)
dbconf, err := NewDBConf("../../db-sample", "environment_variable_config", "")
if err != nil {
t.Fatal(err)
}
got := dbconf.Driver.Name
want := "sqlite3"
if got != want {
t.Errorf("Not able to read the driver type from environment variable."+
"got %v want %v", got, want)
}
gotOpenString := dbconf.Driver.OpenStr
wantOpenString := databaseOpenStringVal
if gotOpenString != wantOpenString {
t.Errorf("Not able to read the open string from the environment."+
"got %v want %v", gotOpenString, wantOpenString)
}
}

402
lib/goose/migrate.go Normal file
View File

@ -0,0 +1,402 @@
package goose
import (
"errors"
"fmt"
"log"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"text/template"
"time"
"github.com/jinzhu/gorm"
_ "github.com/jinzhu/gorm/dialects/mysql"
_ "github.com/jinzhu/gorm/dialects/postgres"
_ "github.com/jinzhu/gorm/dialects/sqlite"
)
var (
ErrTableDoesNotExist = errors.New("table does not exist")
ErrNoPreviousVersion = errors.New("no previous version found")
)
type MigrationRecord struct {
ID uint `gorm:"primary_key"`
VersionId int64
TStamp time.Time `gorm:"default: now()"`
IsApplied bool // was this a result of up() or down()
}
type Migration struct {
Version int64
Next int64 // next version, or -1 if none
Previous int64 // previous version, -1 if none
Source string // path to .go or .sql script
}
type migrationSorter []*Migration
// helpers so we can use pkg sort
func (ms migrationSorter) Len() int { return len(ms) }
func (ms migrationSorter) Swap(i, j int) { ms[i], ms[j] = ms[j], ms[i] }
func (ms migrationSorter) Less(i, j int) bool { return ms[i].Version < ms[j].Version }
func newMigration(v int64, src string) *Migration {
return &Migration{v, -1, -1, src}
}
func RunMigrations(conf *DBConf, migrationsDir string, target int64) (err error) {
db, err := OpenDBFromDBConf(conf)
if err != nil {
return err
}
defer db.Close()
return RunMigrationsOnDb(conf, migrationsDir, target, db)
}
// Runs migration on a specific database instance.
func RunMigrationsOnDb(conf *DBConf, migrationsDir string, target int64, db *gorm.DB) (err error) {
current, err := EnsureDBVersion(conf, db)
if err != nil {
return err
}
migrations, err := CollectMigrations(migrationsDir, current, target)
if err != nil {
return err
}
if len(migrations) == 0 {
fmt.Printf("goose: no migrations to run. current version: %d\n", current)
return nil
}
ms := migrationSorter(migrations)
direction := current < target
ms.Sort(direction)
fmt.Printf("goose: migrating db environment '%v', current version: %d, target: %d\n",
conf.Env, current, target)
for _, m := range ms {
switch filepath.Ext(m.Source) {
case ".go":
err = runGoMigration(conf, m.Source, m.Version, direction)
case ".sql":
err = runSQLMigration(conf, db, m.Source, m.Version, direction)
}
if err != nil {
return errors.New(fmt.Sprintf("FAIL %v, quitting migration", err))
}
fmt.Println("OK ", filepath.Base(m.Source))
}
return nil
}
// collect all the valid looking migration scripts in the
// migrations folder, and key them by version
func CollectMigrations(dirpath string, current, target int64) (m []*Migration, err error) {
// extract the numeric component of each migration,
// filter out any uninteresting files,
// and ensure we only have one file per migration version.
filepath.Walk(dirpath, func(name string, info os.FileInfo, err error) error {
if v, e := NumericComponent(name); e == nil {
for _, g := range m {
if v == g.Version {
log.Fatalf("more than one file specifies the migration for version %d (%s and %s)",
v, g.Source, filepath.Join(dirpath, name))
}
}
if versionFilter(v, current, target) {
m = append(m, newMigration(v, name))
}
}
return nil
})
return m, nil
}
func versionFilter(v, current, target int64) bool {
if target > current {
return v > current && v <= target
}
if target < current {
return v <= current && v > target
}
return false
}
func (ms migrationSorter) Sort(direction bool) {
// sort ascending or descending by version
if direction {
sort.Sort(ms)
} else {
sort.Sort(sort.Reverse(ms))
}
// now that we're sorted in the appropriate direction,
// populate next and previous for each migration
for i, m := range ms {
prev := int64(-1)
if i > 0 {
prev = ms[i-1].Version
ms[i-1].Next = m.Version
}
ms[i].Previous = prev
}
}
// look for migration scripts with names in the form:
// XXX_descriptivename.ext
// where XXX specifies the version number
// and ext specifies the type of migration
func NumericComponent(name string) (int64, error) {
base := filepath.Base(name)
if ext := filepath.Ext(base); ext != ".go" && ext != ".sql" {
return 0, errors.New("not a recognized migration file type")
}
idx := strings.Index(base, "_")
if idx < 0 {
return 0, errors.New("no separator found")
}
n, e := strconv.ParseInt(base[:idx], 10, 64)
if e == nil && n <= 0 {
return 0, errors.New("migration IDs must be greater than zero")
}
return n, e
}
// EnsureDBVersion retrieve the current version for this DB.
// Create and initialize the DB version table if it doesn't exist.
func EnsureDBVersion(conf *DBConf, db *gorm.DB) (int64, error) {
rows := []MigrationRecord{}
err := db.Order("id desc").Find(&rows).Error
if err != nil {
return 0, createVersionTable(conf, db)
}
// The most recent record for each migration specifies
// whether it has been applied or rolled back.
// The first version we find that has been applied is the current version.
toSkip := make([]int64, 0)
for _, row := range rows {
// have we already marked this version to be skipped?
skip := false
for _, v := range toSkip {
if v == row.VersionId {
skip = true
break
}
}
if skip {
continue
}
// if version has been applied we're done
if row.IsApplied {
return row.VersionId, nil
}
// latest version of migration has not been applied.
toSkip = append(toSkip, row.VersionId)
}
panic("failure in EnsureDBVersion()")
}
// Create the goose_db_version table
// and insert the initial 0 value into it
func createVersionTable(conf *DBConf, db *gorm.DB) error {
txn := db.Begin()
if txn.Error != nil {
return txn.Error
}
if err := txn.CreateTable(&MigrationRecord{}).Error; err != nil {
txn.Rollback()
return err
}
record := MigrationRecord{VersionId: 0, IsApplied: true}
if err := txn.Create(&record).Error; err != nil {
txn.Rollback()
return err
}
return txn.Commit().Error
}
// wrapper for EnsureDBVersion for callers that don't already have
// their own DB instance
func GetDBVersion(conf *DBConf) (version int64, err error) {
db, err := OpenDBFromDBConf(conf)
if err != nil {
return -1, err
}
defer db.Close()
version, err = EnsureDBVersion(conf, db)
if err != nil {
return -1, err
}
return version, nil
}
func GetPreviousDBVersion(dirpath string, version int64) (previous int64, err error) {
previous = -1
sawGivenVersion := false
filepath.Walk(dirpath, func(name string, info os.FileInfo, walkerr error) error {
if !info.IsDir() {
if v, e := NumericComponent(name); e == nil {
if v > previous && v < version {
previous = v
}
if v == version {
sawGivenVersion = true
}
}
}
return nil
})
if previous == -1 {
if sawGivenVersion {
// the given version is (likely) valid but we didn't find
// anything before it.
// 'previous' must reflect that no migrations have been applied.
previous = 0
} else {
err = ErrNoPreviousVersion
}
}
return
}
// helper to identify the most recent possible version
// within a folder of migration scripts
func GetMostRecentDBVersion(dirpath string) (version int64, err error) {
version = -1
filepath.Walk(dirpath, func(name string, info os.FileInfo, walkerr error) error {
if walkerr != nil {
return walkerr
}
if !info.IsDir() {
if v, e := NumericComponent(name); e == nil {
if v > version {
version = v
}
}
}
return nil
})
if version == -1 {
err = errors.New("no valid version found")
}
return
}
func CreateMigration(name, migrationType, dir string, t time.Time) (path string, err error) {
if migrationType != "go" && migrationType != "sql" {
return "", errors.New("migration type must be 'go' or 'sql'")
}
timestamp := t.Format("20060102150405")
filename := fmt.Sprintf("%v_%v.%v", timestamp, name, migrationType)
fpath := filepath.Join(dir, filename)
var tmpl *template.Template
if migrationType == "sql" {
tmpl = sqlMigrationTemplate
} else {
tmpl = goMigrationTemplate
}
path, err = writeTemplateToFile(fpath, tmpl, timestamp)
return
}
// FinalizeMigration update the version table for the given migration,
// and finalize the transaction.
func FinalizeMigration(conf *DBConf, txn *gorm.DB, direction bool, v int64) error {
// XXX: drop goose_db_version table on some minimum version number?
record := MigrationRecord{VersionId: v, IsApplied: direction}
if err := txn.Create(&record).Error; err != nil {
txn.Rollback()
return err
}
return txn.Commit().Error
}
var goMigrationTemplate = template.Must(template.New("goose.go-migration").Parse(`
package main
import (
"github.com/jinzhu/gorm"
)
// Up is executed when this migration is applied
func Up_{{ . }}(txn *gorm.DB) {
}
// Down is executed when this migration is rolled back
func Down_{{ . }}(txn *gorm.DB) {
}
`))
var sqlMigrationTemplate = template.Must(template.New("goose.sql-migration").Parse(`
-- +goose Up
-- SQL in section 'Up' is executed when this migration is applied
-- +goose Down
-- SQL section 'Down' is executed when this migration is rolled back
`))

71
lib/goose/migrate_test.go Normal file
View File

@ -0,0 +1,71 @@
package goose
import (
"testing"
)
func TestMigrationMapSortUp(t *testing.T) {
ms := migrationSorter{}
// insert in any order
ms = append(ms, newMigration(20120000, "test"))
ms = append(ms, newMigration(20128000, "test"))
ms = append(ms, newMigration(20129000, "test"))
ms = append(ms, newMigration(20127000, "test"))
ms.Sort(true) // sort Upwards
sorted := []int64{20120000, 20127000, 20128000, 20129000}
validateMigrationSort(t, ms, sorted)
}
func TestMigrationMapSortDown(t *testing.T) {
ms := migrationSorter{}
// insert in any order
ms = append(ms, newMigration(20120000, "test"))
ms = append(ms, newMigration(20128000, "test"))
ms = append(ms, newMigration(20129000, "test"))
ms = append(ms, newMigration(20127000, "test"))
ms.Sort(false) // sort Downwards
sorted := []int64{20129000, 20128000, 20127000, 20120000}
validateMigrationSort(t, ms, sorted)
}
func validateMigrationSort(t *testing.T, ms migrationSorter, sorted []int64) {
for i, m := range ms {
if sorted[i] != m.Version {
t.Error("incorrect sorted version")
}
var next, prev int64
if i == 0 {
prev = -1
next = ms[i+1].Version
} else if i == len(ms)-1 {
prev = ms[i-1].Version
next = -1
} else {
prev = ms[i-1].Version
next = ms[i+1].Version
}
if m.Next != next {
t.Errorf("mismatched Next. v: %v, got %v, wanted %v\n", m, m.Next, next)
}
if m.Previous != prev {
t.Errorf("mismatched Previous v: %v, got %v, wanted %v\n", m, m.Previous, prev)
}
}
t.Log(ms)
}

129
lib/goose/migration_go.go Normal file
View File

@ -0,0 +1,129 @@
package goose
import (
"bytes"
"encoding/gob"
"fmt"
"io/ioutil"
"log"
"os"
"os/exec"
"path/filepath"
"text/template"
)
type templateData struct {
Version int64
Import string
Conf string // gob encoded DBConf
Direction bool
Func string
}
//
// Run a .go migration.
//
// In order to do this, we copy a modified version of the
// original .go migration, and execute it via `go run` along
// with a main() of our own creation.
//
func runGoMigration(conf *DBConf, path string, version int64, direction bool) error {
// everything gets written to a temp dir, and zapped afterwards
d, e := ioutil.TempDir("", "goose")
if e != nil {
log.Fatal(e)
}
defer os.RemoveAll(d)
directionStr := "Down"
if direction {
directionStr = "Up"
}
var bb bytes.Buffer
if err := gob.NewEncoder(&bb).Encode(conf); err != nil {
return err
}
// XXX: there must be a better way of making this byte array
// available to the generated code...
// but for now, print an array literal of the gob bytes
var sb bytes.Buffer
sb.WriteString("[]byte{ ")
for _, b := range bb.Bytes() {
sb.WriteString(fmt.Sprintf("0x%02x, ", b))
}
sb.WriteString("}")
td := &templateData{
Version: version,
Import: conf.Driver.Import,
Conf: sb.String(),
Direction: direction,
Func: fmt.Sprintf("%v_%v", directionStr, version),
}
main, e := writeTemplateToFile(filepath.Join(d, "goose_main.go"), goMigrationDriverTemplate, td)
if e != nil {
log.Fatal(e)
}
outpath := filepath.Join(d, filepath.Base(path))
if _, e = copyFile(outpath, path); e != nil {
log.Fatal(e)
}
cmd := exec.Command("go", "run", main, outpath)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if e = cmd.Run(); e != nil {
log.Fatal("`go run` failed: ", e)
}
return nil
}
//
// template for the main entry point to a go-based migration.
// this gets linked against the substituted versions of the user-supplied
// scripts in order to execute a migration via `go run`
//
var goMigrationDriverTemplate = template.Must(template.New("goose.go-driver").Parse(`
package main
import (
"log"
"bytes"
"encoding/gob"
_ "{{.Import}}"
"github.com/Altoros/gorm-goose/lib/goose"
)
func main() {
var conf goose.DBConf
buf := bytes.NewBuffer({{ .Conf }})
if err := gob.NewDecoder(buf).Decode(&conf); err != nil {
log.Fatal("gob.Decode - ", err)
}
db, err := goose.OpenDBFromDBConf(&conf)
if err != nil {
log.Fatal("failed to open DB:", err)
}
defer db.Close()
txn := db.Begin()
if txn.Error != nil {
log.Fatal("db.Begin:", txn.Error)
}
{{ .Func }}(txn)
err = goose.FinalizeMigration(&conf, txn, {{ .Direction }}, {{ .Version }})
if err != nil {
log.Fatal("Commit() failed:", err)
}
}
`))

168
lib/goose/migration_sql.go Normal file
View File

@ -0,0 +1,168 @@
package goose
import (
"bufio"
"bytes"
"io"
"log"
"os"
"path/filepath"
"strings"
"github.com/jinzhu/gorm"
)
const sqlCmdPrefix = "-- +goose "
// Checks the line to see if the line has a statement-ending semicolon
// or if the line contains a double-dash comment.
func endsWithSemicolon(line string) bool {
prev := ""
scanner := bufio.NewScanner(strings.NewReader(line))
scanner.Split(bufio.ScanWords)
for scanner.Scan() {
word := scanner.Text()
if strings.HasPrefix(word, "--") {
break
}
prev = word
}
return strings.HasSuffix(prev, ";")
}
// Split the given sql script into individual statements.
//
// The base case is to simply split on semicolons, as these
// naturally terminate a statement.
//
// However, more complex cases like pl/pgsql can have semicolons
// within a statement. For these cases, we provide the explicit annotations
// 'StatementBegin' and 'StatementEnd' to allow the script to
// tell us to ignore semicolons.
func splitSQLStatements(r io.Reader, direction bool) (stmts []string) {
var buf bytes.Buffer
scanner := bufio.NewScanner(r)
// track the count of each section
// so we can diagnose scripts with no annotations
upSections := 0
downSections := 0
statementEnded := false
ignoreSemicolons := false
directionIsActive := false
for scanner.Scan() {
line := scanner.Text()
// handle any goose-specific commands
if strings.HasPrefix(line, sqlCmdPrefix) {
cmd := strings.TrimSpace(line[len(sqlCmdPrefix):])
switch cmd {
case "Up":
directionIsActive = (direction == true)
upSections++
break
case "Down":
directionIsActive = (direction == false)
downSections++
break
case "StatementBegin":
if directionIsActive {
ignoreSemicolons = true
}
break
case "StatementEnd":
if directionIsActive {
statementEnded = (ignoreSemicolons == true)
ignoreSemicolons = false
}
break
}
}
if !directionIsActive {
continue
}
if _, err := buf.WriteString(line + "\n"); err != nil {
log.Fatalf("io err: %v", err)
}
// Wrap up the two supported cases: 1) basic with semicolon; 2) psql statement
// Lines that end with semicolon that are in a statement block
// do not conclude statement.
if (!ignoreSemicolons && endsWithSemicolon(line)) || statementEnded {
statementEnded = false
stmts = append(stmts, buf.String())
buf.Reset()
}
}
if err := scanner.Err(); err != nil {
log.Fatalf("scanning migration: %v", err)
}
// diagnose likely migration script errors
if ignoreSemicolons {
log.Println("WARNING: saw '-- +goose StatementBegin' with no matching '-- +goose StatementEnd'")
}
if bufferRemaining := strings.TrimSpace(buf.String()); len(bufferRemaining) > 0 {
log.Printf("WARNING: Unexpected unfinished SQL query: %s. Missing a semicolon?\n", bufferRemaining)
}
if upSections == 0 && downSections == 0 {
log.Fatal("ERROR: no Up/Down annotations found, so no statements were executed.")
}
return
}
// Run a migration specified in raw SQL.
//
// Sections of the script can be annotated with a special comment,
// starting with "-- +goose" to specify whether the section should
// be applied during an Up or Down migration
//
// All statements following an Up or Down directive are grouped together
// until another direction directive is found.
func runSQLMigration(conf *DBConf, db *gorm.DB, scriptFile string, v int64, direction bool) error {
txn := db.Begin()
if txn.Error != nil {
log.Fatal("db.Begin:", txn.Error)
}
f, err := os.Open(scriptFile)
if err != nil {
log.Fatal(err)
}
// find each statement, checking annotations for up/down direction
// and execute each of them in the current transaction.
// Commits the transaction if successfully applied each statement and
// records the version into the version table or returns an error and
// rolls back the transaction.
for _, query := range splitSQLStatements(f, direction) {
if err = txn.Exec(query).Error; err != nil {
txn.Rollback()
log.Fatalf("FAIL %s (%v), quitting migration.", filepath.Base(scriptFile), err)
return err
}
}
if err = FinalizeMigration(conf, txn, direction, v); err != nil {
log.Fatalf("error finalizing migration %s, quitting. (%v)", filepath.Base(scriptFile), err)
}
return nil
}

147
lib/goose/migration_sql_test.go Normal file
View File

@ -0,0 +1,147 @@
package goose
import (
"strings"
"testing"
)
func TestSemicolons(t *testing.T) {
type testData struct {
line string
result bool
}
tests := []testData{
{
line: "END;",
result: true,
},
{
line: "END; -- comment",
result: true,
},
{
line: "END ; -- comment",
result: true,
},
{
line: "END -- comment",
result: false,
},
{
line: "END -- comment ;",
result: false,
},
{
line: "END \" ; \" -- comment",
result: false,
},
}
for _, test := range tests {
r := endsWithSemicolon(test.line)
if r != test.result {
t.Errorf("incorrect semicolon. got %v, want %v", r, test.result)
}
}
}
func TestSplitStatements(t *testing.T) {
type testData struct {
sql string
direction bool
count int
}
tests := []testData{
{
sql: functxt,
direction: true,
count: 2,
},
{
sql: functxt,
direction: false,
count: 2,
},
{
sql: multitxt,
direction: true,
count: 2,
},
{
sql: multitxt,
direction: false,
count: 2,
},
}
for _, test := range tests {
stmts := splitSQLStatements(strings.NewReader(test.sql), test.direction)
if len(stmts) != test.count {
t.Errorf("incorrect number of stmts. got %v, want %v", len(stmts), test.count)
}
}
}
var functxt = `-- +goose Up
CREATE TABLE IF NOT EXISTS histories (
id BIGSERIAL PRIMARY KEY,
current_value varchar(2000) NOT NULL,
created_at timestamp with time zone NOT NULL
);
-- +goose StatementBegin
CREATE OR REPLACE FUNCTION histories_partition_creation( DATE, DATE )
returns void AS $$
DECLARE
create_query text;
BEGIN
FOR create_query IN SELECT
'CREATE TABLE IF NOT EXISTS histories_'
|| TO_CHAR( d, 'YYYY_MM' )
|| ' ( CHECK( created_at >= timestamp '''
|| TO_CHAR( d, 'YYYY-MM-DD 00:00:00' )
|| ''' AND created_at < timestamp '''
|| TO_CHAR( d + INTERVAL '1 month', 'YYYY-MM-DD 00:00:00' )
|| ''' ) ) inherits ( histories );'
FROM generate_series( $1, $2, '1 month' ) AS d
LOOP
EXECUTE create_query;
END LOOP; -- LOOP END
END; -- FUNCTION END
$$
language plpgsql;
-- +goose StatementEnd
-- +goose Down
drop function histories_partition_creation(DATE, DATE);
drop TABLE histories;
`
// test multiple up/down transitions in a single script
var multitxt = `-- +goose Up
CREATE TABLE post (
id int NOT NULL,
title text,
body text,
PRIMARY KEY(id)
);
-- +goose Down
DROP TABLE post;
-- +goose Up
CREATE TABLE fancier_post (
id int NOT NULL,
title text,
body text,
created_on timestamp without time zone,
PRIMARY KEY(id)
);
-- +goose Down
DROP TABLE fancier_post;
`

40
lib/goose/util.go Normal file
View File

@ -0,0 +1,40 @@
package goose
import (
"io"
"os"
"text/template"
)
// common routines
func writeTemplateToFile(path string, t *template.Template, data interface{}) (string, error) {
f, e := os.Create(path)
if e != nil {
return "", e
}
defer f.Close()
e = t.Execute(f, data)
if e != nil {
return "", e
}
return f.Name(), nil
}
func copyFile(dst, src string) (int64, error) {
sf, err := os.Open(src)
if err != nil {
return 0, err
}
defer sf.Close()
df, err := os.Create(dst)
if err != nil {
return 0, err
}
defer df.Close()
return io.Copy(df, sf)
}