diff --git a/app/Commands/Database/ImportCommand.php b/app/Commands/Database/ImportCommand.php index c4a35e9..1e66a72 100644 --- a/app/Commands/Database/ImportCommand.php +++ b/app/Commands/Database/ImportCommand.php @@ -76,7 +76,7 @@ public function handle(): void Pdo::validateConfiguration(); - $tmpFilePath = tempnam(Directory::getTmpDirectory(), 'roost_'); + $tmpFilePath = tempnam(Directory::getTmpDirectory(), 'roost_tmp_dump_'); $pipeUnarchive = new Pipe(); $isUnarchive = Archive::addUnarchiveCommand($dbPath, $pipeUnarchive); if ($isUnarchive) { diff --git a/app/Commands/Database/ListCommand.php b/app/Commands/Database/ListCommand.php index 1d43b3c..734f458 100644 --- a/app/Commands/Database/ListCommand.php +++ b/app/Commands/Database/ListCommand.php @@ -43,13 +43,13 @@ public function handle(): void }); } - $tableRows = array_map(static function ($dbName) { + $dbRows = array_map(static function ($dbName) { return [$dbName]; }, $dbList); $table = new Table($this->output); $table->setHeaders(['Databases']); - $table->setRows($tableRows); + $table->setRows($dbRows); $table->render(); } } diff --git a/app/Commands/Dump/DownloadCommand.php b/app/Commands/Dump/DownloadCommand.php index 5a90f99..0becea9 100644 --- a/app/Commands/Dump/DownloadCommand.php +++ b/app/Commands/Dump/DownloadCommand.php @@ -10,6 +10,7 @@ use App\Services\AwsS3; use App\Services\Dump; use App\Commands\Database\ImportCommand; +use App\Commands\Warden\ImportCommand as WardenImportCommand; class DownloadCommand extends Command { @@ -23,6 +24,7 @@ class DownloadCommand extends Command protected $signature = self::COMMAND . ' {dump? : Dump file name}' . ' {--i|import : Import downloaded dump}' + . ' {--w|import-warden : Import downloaded dump into Warden Db container}' . ' {--r|remove-file : Remove file after import}' . ' {--no-progress : Do not display progress}' . ' {--f|force : Overwrite local file if exits without confirmation}'; @@ -90,6 +92,19 @@ public function handle(): void ] ); + $this->processDeletingFile($dbFile); + } elseif ($this->option('import-warden')) { + $this->info('Import the dump:'); + + $this->call( + WardenImportCommand::COMMAND, + [ + 'file' => $dbFile, + '--no-progress' => $this->option('no-progress'), + '--quiet' => $this->option('quiet'), + ] + ); + $this->processDeletingFile($dbFile); } } diff --git a/app/Commands/Warden/BackupCommand.php b/app/Commands/Warden/BackupCommand.php new file mode 100644 index 0000000..4bfa34c --- /dev/null +++ b/app/Commands/Warden/BackupCommand.php @@ -0,0 +1,83 @@ +call( + ExportCommand::COMMAND, + [ + 'file' => $this->argument('file'), + '--magento-directory' => $this->option('magento-directory'), + '--db-host' => $this->option('db-host'), + '--db-port' => $this->option('db-port'), + '--db-name' => $this->option('db-name'), + '--db-username' => $this->option('db-username'), + '--db-password' => $this->option('db-password'), + '--storage' => $this->option('storage'), + '--aws-bucket' => $this->option('aws-bucket'), + '--aws-access-key' => $this->option('aws-access-key'), + '--aws-secret-key' => $this->option('aws-secret-key'), + '--aws-region' => $this->option('aws-region'), + '--tag' => $this->option('tag'), + '--strip' => $this->option('strip'), + '--compatibility' => $this->option('compatibility'), + '--no-progress' => $this->option('no-progress'), + '--print' => $this->option('print'), + '--skip-filter' => $this->option('skip-filter'), + '--project' => $this->option('project'), + '--force' => $this->option('force'), + '--quiet' => $this->option('quiet'), + '--upload' => true, + '--remove-file' => !$this->option('keep-file') + ] + ); + + $clean = (int)$this->option('clean'); + if ($clean > 0) { + $this->call( + CleanCommand::COMMAND, + [ + 'count' => $clean, + '--project' => $this->option('project'), + '--tag' => $this->option('tag'), + ] + ); + } + } +} diff --git a/app/Commands/Warden/CreateCommand.php b/app/Commands/Warden/CreateCommand.php new file mode 100644 index 0000000..6e4baab --- /dev/null +++ b/app/Commands/Warden/CreateCommand.php @@ -0,0 +1,61 @@ +argument('name'); + + if ($this->option('force')) { + $this->call(DropCommand::COMMAND, ['name' => $dbName]); + } + + $taskMessage = $dbName + ? sprintf('Create DB %s if not exists', $dbName) + : 'Create DB if not exists'; + + $this->task($taskMessage, static function () use ($dbName) { + try { + $wardenCommand = WardenDatabase::createWardenDbCommand('create'); + if ($dbName) { + $wardenCommand->argument($dbName); + } + $wardenCommand->exec(); + + $result = true; + } catch (\Symfony\Component\Process\Exception\ProcessFailedException $e) { + $result = $e->getProcess()->getCommandLine(); + } catch (\Exception $e) { + $result = $e->getMessage(); + } + return $result; + }); + } +} diff --git a/app/Commands/Warden/DropCommand.php b/app/Commands/Warden/DropCommand.php new file mode 100644 index 0000000..395b323 --- /dev/null +++ b/app/Commands/Warden/DropCommand.php @@ -0,0 +1,57 @@ +argument('name'); + + $taskMessage = $dbName + ? sprintf('Drop DB "%s" if exists', $dbName) + : 'Drop DB if exists'; + + $this->task($taskMessage, static function () use ($dbName) { + try { + + $wardenCommand = WardenDatabase::createWardenDbCommand('drop'); + if ($dbName) { + $wardenCommand->argument($dbName); + } + $wardenCommand->exec(); + + $result = true; + } catch (\Symfony\Component\Process\Exception\ProcessFailedException $e) { + $result = $e->getProcess()->getCommandLine(); + } catch (\Exception $e) { + $result = $e->getMessage(); + } + return $result; + }); + } +} diff --git a/app/Commands/Warden/ExportCommand.php b/app/Commands/Warden/ExportCommand.php new file mode 100644 index 0000000..de384ca --- /dev/null +++ b/app/Commands/Warden/ExportCommand.php @@ -0,0 +1,208 @@ +argument('file'); + if (empty($fileName)) { + $defaultName = $this->getDefaultDumpName( + (AppConfig::getConfigValue('project') ?: $dbName), + $this->option('tag') + ); + $fileName = $this->option('no-interaction') ? $defaultName : $this->getDumpName($defaultName); + } + $dumpPath = Dump::getDumpPath($this->updateDumpExtension($fileName)); + + $strip = (string)$this->option('strip'); + $stripTableList = !empty($strip) ? DbStrip::getTableList($strip, WardenDatabase::getAllTables($dbName)) : []; + if (!empty($stripTableList)) { + $wardenDumpCommand = WardenDatabase::createWardenDbCommand('dump'); + $wardenDumpCommand->arguments([ + $dbName, + '--single-transaction', + '--quick', + '--default-character-set=utf8', + '--add-drop-table', + '--no-data' + ]); + $wardenDumpCommand->arguments($stripTableList); + + if ($this->option('compatibility')) { + $wardenDumpCommand->arguments([ + '--set-gtid-purged=OFF' + ]); + } + + $structurePipe = new Pipe(); + $structurePipe->command($wardenDumpCommand); + + if (!$this->option('skip-filter')) { + $structurePipe->commands(Database::getFilterCommands()); + } + + Archive::addArchiveCommand($dumpPath, $structurePipe); + + $structurePipe->getLastCommand()->output($dumpPath); + + if ($this->option('print')) { + $this->line($structurePipe->toString()); + } else { + $structurePipe->passthru(); + } + } + + $wardenDumpCommand = WardenDatabase::createWardenDbCommand('dump'); + $wardenDumpCommand->arguments([ + $dbName, + '--single-transaction', + '--quick', + '--routines=true', + '--add-drop-table', + '--default-character-set=utf8' + ]); + if ($this->option('compatibility')) { + $wardenDumpCommand->arguments([ + '--set-gtid-purged=OFF' + ]); + } + + $pipe = new Pipe(); + $pipe->command($wardenDumpCommand); + + if (!empty($stripTableList)) { + $stripTableArguments = array_map(static function ($table) use ($dbName) { + return sprintf('--ignore-table=%s', $dbName . '.' . $table); + }, $stripTableList); + $pipe->getLastCommand()->arguments($stripTableArguments); + } + + if (!$this->option('no-progress') && !$this->option('quiet') && Progress::isPvAvailable()) { + $pipe->command( + (new Pv)->arguments(['-b', '-t', '-w', '80', '-N', 'Export']) + ); + } + + if (!$this->option('skip-filter')) { + $pipe->commands(Database::getFilterCommands()); + } + + Archive::addArchiveCommand($dumpPath, $pipe); + + $pipe->getLastCommand()->output($dumpPath, !empty($stripTableList)); + + if ($this->option('print')) { + $this->line($pipe->toString()); + } else { + $pipe->passthru(); + $this->comment(sprintf('DB %s is exported to %s', $dbName, $dumpPath)); + } + + if (!$this->option('print') && $this->option('upload')) { + $this->info('Upload the dump:'); + + $this->call( + UploadCommand::COMMAND, + [ + 'file' => $dumpPath, + '--project' => $this->option('project'), + '--no-progress' => $this->option('no-progress'), + '--force' => $this->option('force'), + '--quiet' => $this->option('quiet'), + ] + ); + + $this->processDeletingFile($dumpPath); + } + } + + /** + * @param string $defaultName + * @return string + */ + private function getDumpName(string $defaultName): string + { + return $this->ask('Enter Dump file name (location)', $defaultName); + } + + /** + * @param string $identifier + * @param string|null $tag + * @return string + */ + private function getDefaultDumpName(string $identifier, ?string $tag = null): string + { + $tagSuffix = !empty($tag) ? '[' . $tag . ']' : ''; + return $identifier . '-' . gmdate('Y.m.d') . '-' . gmdate('H.i.s') . $tagSuffix . '.sql.gz'; + } + + /** + * @param string $file + * @return string + */ + private function updateDumpExtension(string $file): string + { + return DumpFile::isOutcomeFileSupported($file) ? $file : $file . '.sql.gz'; + } + + /** + * @param string $dumpPath + * @return void + */ + private function processDeletingFile(string $dumpPath): void + { + if (!$this->option('remove-file')) { + return; + } + + File::delete($dumpPath); + } +} diff --git a/app/Commands/Warden/ImportCommand.php b/app/Commands/Warden/ImportCommand.php new file mode 100644 index 0000000..59368e6 --- /dev/null +++ b/app/Commands/Warden/ImportCommand.php @@ -0,0 +1,147 @@ +argument('file'); + $fileName = $fileName || $this->option('quiet') ? $fileName : Dump::getDumpName('Import DB'); + if ($fileName === null) { + $this->error('Dump file is not specified.'); + return; + } + + $dbPath = Dump::getDumpPath($fileName); + if (!$this->verifyPath($dbPath)) { + $this->error(sprintf('Passed path does not exist or not a file: %s', $dbPath)); + return; + } + $originDbPath = $dbPath; + + $fileType = File::extension($fileName); + if (!DumpFile::isIncomeFileSupported($fileName)) { + $this->error(sprintf('The file type is not supported: %s', $fileType)); + return; + } + + if (!$this->option('print')) { + $this->call(CreateCommand::COMMAND, ['name' => $dbName, '--force' => true]); + } + + $tmpFilePath = tempnam(Directory::getTmpDirectory(), 'roost_tmp_dump_'); + $pipeUnarchive = new Pipe(); + $isUnarchive = Archive::addUnarchiveCommand($dbPath, $pipeUnarchive); + if ($isUnarchive) { + + if (!$this->option('no-progress') && !$this->option('quiet') && Progress::isPvAvailable()) { + $pipeUnarchive->command( + (new Pv)->arguments(['-b', '-t', '-w', '80', '-N', 'Unpack']) + ); + } + + $pipeUnarchive->getLastCommand()->output($tmpFilePath); + if ($this->option('print')) { + $this->line($pipeUnarchive->toString()); + } else { + $pipeUnarchive->passthru(); + } + + $dbPath = $tmpFilePath; + } + + + $pipe = new Pipe(); + + if (!$this->option('no-progress') && !$this->option('quiet') && Progress::isPvAvailable()) { + $pipe->command( + (new Pv)->arguments([$dbPath, '-w', '80', '-N', 'Import']) + ); + } else { + $pipe->command( + (new Cat)->argument($dbPath) + ); + } + + if (!$this->option('skip-filter')) { + $pipe->commands(Database::getFilterCommands()); + } + + $pipe->command($this->createWardenDbImport($dbName)); + + if ($this->option('print')) { + $this->line($pipe->toString()); + return; + } + + $pipe->passthru(); + + File::delete($tmpFilePath); + + $this->comment(sprintf('DB %s is imported from dump %s', $dbName, $originDbPath)); + } + + /** + * @param string $dbPath + * @return bool + */ + private function verifyPath(string $dbPath): bool + { + return File::exists($dbPath) && File::isFile($dbPath); + } + + /** + * @param string|null $dbName + * @return \App\Shell\Command\Warden + */ + private function createWardenDbImport(string $dbName = null): \App\Shell\Command\Warden + { + $wardenCommand = WardenDatabase::createWardenDbCommand('import'); + if ($dbName) { + $wardenCommand->argument($dbName); + } + $wardenCommand->argument('--force'); + return $wardenCommand; + } +} diff --git a/app/Commands/Warden/ListCommand.php b/app/Commands/Warden/ListCommand.php new file mode 100644 index 0000000..2956adc --- /dev/null +++ b/app/Commands/Warden/ListCommand.php @@ -0,0 +1,52 @@ +argument('search'); + + $dbList = WardenDatabase::getExistingDatabases(); + if (!empty($search)) { + $dbList = array_filter($dbList, static function ($dbName) use ($search) { + return strpos($dbName, $search) !== false; + }); + } + + $dbRows = array_map(static function ($dbName) { + return [$dbName]; + }, $dbList); + + $table = new Table($this->output); + $table->setHeaders(['Databases']); + $table->setRows($dbRows); + $table->render(); + } +} diff --git a/app/Commands/Warden/RestoreCommand.php b/app/Commands/Warden/RestoreCommand.php new file mode 100644 index 0000000..aa32ae1 --- /dev/null +++ b/app/Commands/Warden/RestoreCommand.php @@ -0,0 +1,92 @@ +argument('dump'); + if (empty($dump) && $this->option('most-recent')) { + $project = AppConfig::getConfigValue('project'); + if (empty($project)) { + $this->error('Project is not specified.'); + return; + } + + $tag = $this->option('tag'); + $dump = $this->getMostRecentDump($project, $tag); + if (empty($dump)) { + $tagInfo = $tag ? sprintf(' and [%s] tag', $tag) : ''; + $this->error(sprintf('There is not found dump for %s project%s.', $project, $tagInfo)); + return; + } + } + + $this->call( + DownloadCommand::COMMAND, + [ + 'dump' => $dump, + '--magento-directory' => $this->option('magento-directory'), + '--db-host' => $this->option('db-host'), + '--db-port' => $this->option('db-port'), + '--db-name' => $this->option('db-name'), + '--db-username' => $this->option('db-username'), + '--db-password' => $this->option('db-password'), + '--storage' => $this->option('storage'), + '--aws-bucket' => $this->option('aws-bucket'), + '--aws-access-key' => $this->option('aws-access-key'), + '--aws-secret-key' => $this->option('aws-secret-key'), + '--aws-region' => $this->option('aws-region'), + '--project' => $this->option('project'), + '--no-progress' => $this->option('no-progress'), + '--force' => $this->option('force'), + '--quiet' => $this->option('quiet'), + '--import-warden' => true, + '--remove-file' => !$this->option('keep-file') + ] + ); + } + + private function getMostRecentDump(string $project, ?string $tag = null): ?string + { + $initProgress = !$this->option('no-progress') && !$this->option('quiet'); + AwsS3::initAwsBucket($this->output, $initProgress); + + $dumpItems = AwsS3::getAwsProjectDumps($project, $tag); + $dumpItems = array_reverse($dumpItems); + + return $dumpItems[0]['name'] ?? null; + } +} diff --git a/app/Services/WardenDatabase.php b/app/Services/WardenDatabase.php new file mode 100644 index 0000000..a7d2550 --- /dev/null +++ b/app/Services/WardenDatabase.php @@ -0,0 +1,77 @@ +arguments(['-N', '-e', '"SHOW DATABASES"']); + + $output = null; + $wardenCommand->exec($output); + + $dbs = self::parseResult($output); + return array_diff($dbs, static::$systemDbs); + } + + /** + * @param string|null $dbName + * @return array + */ + public static function getAllTables(string $dbName = null): array + { + $wardenCommand = static::createWardenDbCommand('connect'); + if ($dbName) { + $wardenCommand->argument($dbName); + } + $wardenCommand->arguments(['-N', '-e', '"SHOW TABLES"']); + + $output = null; + $wardenCommand->exec($output); + + return self::parseResult($output); + } + + /** + * @param string|null $command + * @return \App\Shell\Command\Warden + */ + public static function createWardenDbCommand(string $command = null): Warden + { + $wardenCommand = new Warden(); + $wardenCommand->argument('db'); + if ($command) { + $wardenCommand->argument($command); + } + return $wardenCommand; + } + + /** + * @param array|null $result + * @return array + */ + private static function parseResult(array $result = null): array + { + $result = !empty($result) ? array_slice($result, 1, -1) : []; + return array_map( + static function ($row) { + return trim(substr($row, 1, -1)); + }, + $result + ); + } +} diff --git a/app/Shell/Command/Base.php b/app/Shell/Command/Base.php index 6aca054..4b88a6f 100644 --- a/app/Shell/Command/Base.php +++ b/app/Shell/Command/Base.php @@ -148,6 +148,16 @@ private function toProcess(): Process return new Process($this->toParts(), null, $this->envVars, null, 60 * 60 * 12); } + /** + * @param array|null $output + * @param int|null $resultCode + * @return string|null + */ + public function exec(array &$output = null, int $resultCode = null): ?string + { + return exec($this->toString(), $output, $resultCode); + } + /** * @return void */ diff --git a/app/Shell/Command/Warden.php b/app/Shell/Command/Warden.php new file mode 100644 index 0000000..6356f7d --- /dev/null +++ b/app/Shell/Command/Warden.php @@ -0,0 +1,16 @@ +