Skip to content

Commit

Permalink
Improve view support for pscale database dump / restore-dump (#932)
Browse files Browse the repository at this point in the history
* Added better support for views within dumper.go

Currently, `dumper.go` is able to export view definitions without issue, but it unfortunately also runs the view and includes, or attempts to include, the data associated with the view too which is unnecessary.

The changes here add support for collecting the list of views so that processing can be slightly adjusted if a "table" is actually a view.

In that situation, the same `-schema-view.sql` file suffix is utilized that the MyDumper project uses for views and the step that unnecessarily runs the view query is skipped to avoid that issue.

* Added better support for views within loader.go

The changes made to `loader.go` add support for collecting any files ending in the view suffix, `-schema-view.sql`, and processing them slightly differently than in `restoreTableSchema()`.

The main change within the new `restoreViews()` method, aside from some variable name adjustments and the use of the `viewSuffix`, is the change to using `DROP VIEW` during the overwrite step to properly allow for deleting the existing views.

Additionally, the creation of the views is set to occur after the creation of the tables to help avoid that dependency issue.

* Adding test fixes for dumper_test.go

Some additional cases for the new `information_schema` query needed to be accounted for in the tests.
  • Loading branch information
orware authored Nov 4, 2024
1 parent 34b78e2 commit a9c9e81
Show file tree
Hide file tree
Showing 3 changed files with 296 additions and 2 deletions.
40 changes: 38 additions & 2 deletions internal/dumper/dumper.go
Original file line number Diff line number Diff line change
Expand Up @@ -107,11 +107,19 @@ func (d *Dumper) Run(ctx context.Context) error {
}

tables := make([][]string, len(databases))
views := make([]map[string]bool, len(databases))
for i, database := range databases {
if d.cfg.Table != "" {
tables[i] = strings.Split(d.cfg.Table, ",")
} else {
tables[i], err = d.allTables(conn, database)

if err != nil {
return err
}

views[i], err = d.allViews(conn, database)

if err != nil {
return err
}
Expand All @@ -135,13 +143,18 @@ func (d *Dumper) Run(ctx context.Context) error {
}

conn := initPool.Get()
err := d.dumpTableSchema(conn, database, table)
err := d.dumpTableSchema(conn, database, table, views[i])
if err != nil {
return err
}

initPool.Put(conn)

if _, ok := views[i][table]; ok {
// If we just processed a view we don't want to dump it so the next part is skipped:
continue
}

conn = pool.Get()
wg.Add(1)
go func(conn *Connection, database string, table string) {
Expand Down Expand Up @@ -200,7 +213,7 @@ func writeMetaData(outdir string) error {
return writeFile(file, "")
}

func (d *Dumper) dumpTableSchema(conn *Connection, database string, table string) error {
func (d *Dumper) dumpTableSchema(conn *Connection, database string, table string, views map[string]bool) error {
qr, err := conn.Fetch(fmt.Sprintf("SHOW CREATE TABLE `%s`.`%s`", database, table))
if err != nil {
return err
Expand All @@ -209,6 +222,11 @@ func (d *Dumper) dumpTableSchema(conn *Connection, database string, table string
schema := qr.Rows[0][1].String() + ";\n"

file := fmt.Sprintf("%s/%s.%s-schema.sql", d.cfg.Outdir, database, table)
if _, ok := views[table]; ok {
// https://github.com/mydumper/mydumper/blob/e55612616d17281a45eed0a60a9b054cdd1fe064/src/myloader_common.c#L374
file = fmt.Sprintf("%s/%s.%s-schema-view.sql", d.cfg.Outdir, database, table)
}

err = writeFile(file, schema)
if err != nil {
return err
Expand Down Expand Up @@ -379,6 +397,24 @@ func (d *Dumper) allTables(conn *Connection, database string) ([]string, error)
return tables, nil
}

func (d *Dumper) allViews(conn *Connection, database string) (map[string]bool, error) {
query := `SELECT TABLE_NAME
FROM information_schema.TABLES
WHERE TABLE_SCHEMA LIKE '%s'
AND TABLE_TYPE = 'VIEW'
`
qr, err := conn.Fetch(fmt.Sprintf(query, database))
if err != nil {
return nil, err
}

views := make(map[string]bool)
for _, t := range qr.Rows {
views[t[0].String()] = true
}
return views, nil
}

func (d *Dumper) allDatabases(conn *Connection) ([]string, error) {
qr, err := conn.Fetch("SHOW DATABASES")
if err != nil {
Expand Down
Loading

0 comments on commit a9c9e81

Please sign in to comment.