From 2e49442155f409378038a9712d236f7342c88e84 Mon Sep 17 00:00:00 2001 From: Lonny Loquesol Date: Fri, 22 Nov 2024 22:15:47 +0100 Subject: [PATCH] Issue #178 - Laravel support --- composer.json | 10 + .../Config/Loader/PhpFileLoader.php | 32 +++ src/Bridge/Laravel/DbToolsServiceProvider.php | 270 ++++++++++++++++++ .../LaravelDatabaseSessionRegistry.php | 84 ++++++ .../Laravel/Resources/config/db-tools.php | 218 ++++++++++++++ 5 files changed, 614 insertions(+) create mode 100644 src/Anonymization/Config/Loader/PhpFileLoader.php create mode 100644 src/Bridge/Laravel/DbToolsServiceProvider.php create mode 100644 src/Bridge/Laravel/LaravelDatabaseSessionRegistry.php create mode 100644 src/Bridge/Laravel/Resources/config/db-tools.php diff --git a/composer.json b/composer.json index dbbf352a..80befff4 100644 --- a/composer.json +++ b/composer.json @@ -27,6 +27,9 @@ "doctrine/doctrine-bundle": "^2.10.0", "doctrine/orm": "^2.15|^3.0", "friendsofphp/php-cs-fixer": "^3.34", + "illuminate/console": "^11.0", + "illuminate/database": "^11.0", + "illuminate/support": "^11.0", "phpstan/phpstan": "^1.10", "phpunit/phpunit": "^10.4", "symfony/dependency-injection": "^6.0|^7.0", @@ -71,5 +74,12 @@ "@phpcs --dry-run", "@phpstan" ] + }, + "extra": { + "laravel": { + "providers": [ + "MakinaCorpus\\DbToolsBundle\\Bridge\\Laravel\\DbToolsServiceProvider" + ] + } } } diff --git a/src/Anonymization/Config/Loader/PhpFileLoader.php b/src/Anonymization/Config/Loader/PhpFileLoader.php new file mode 100644 index 00000000..6b6b77a2 --- /dev/null +++ b/src/Anonymization/Config/Loader/PhpFileLoader.php @@ -0,0 +1,32 @@ +file; + + if (!\is_array($data)) { + throw new ConfigurationException(\sprintf( + "File \"%s\" is not a valid PHP anonymization configuration file (must return an array).", + $this->file + )); + } + + return $data; + } +} diff --git a/src/Bridge/Laravel/DbToolsServiceProvider.php b/src/Bridge/Laravel/DbToolsServiceProvider.php new file mode 100644 index 00000000..6ae6d248 --- /dev/null +++ b/src/Bridge/Laravel/DbToolsServiceProvider.php @@ -0,0 +1,270 @@ +app->extend('config', function (Repository $config, Application $app) { + $dbToolsConfig = $config->get('db-tools', []); + $dbToolsConfig = ['db_tools' => $dbToolsConfig]; + + $processor = new Processor(); + $configuration = new DbToolsConfiguration(false, true); + + $dbToolsConfig = $processor->processConfiguration($configuration, $dbToolsConfig); + $dbToolsConfig = DbToolsConfiguration::fixLegacyOptions($dbToolsConfig); + $dbToolsConfig = DbToolsConfiguration::appendPostConfig($dbToolsConfig); + + $config->set('db-tools', $dbToolsConfig); + + return $config; + }); + + $this->app->singleton(DatabaseSessionRegistry::class, function (Application $app) { + return new LaravelDatabaseSessionRegistry($app->make('db')); + }); + + $this->app->singleton(ConfigurationRegistry::class, function (Application $app) { + /** @var Repository $config */ + $config = $app->make('config'); + + $defaultConfig = new Configuration( + backupBinary: $config->get('db-tools.backup_binary', '/usr/bin/pg_dump'), + backupExcludedTables: $config->get('db-tools.backup_excluded_tables'), + backupExpirationAge: $config->get('db-tools.backup_expiration_age', '3 months ago'), + backupOptions: $config->get('db-tools.backup_options', '-Z 5 --lock-wait-timeout=120'), + backupTimeout: $config->get('db-tools.backup_timeout', 600), + restoreBinary: $config->get('db-tools.restore_binary', '/usr/bin/pg_restore'), + restoreOptions: $config->get('db-tools.restore_options', '-j 2 --clean --if-exists --disable-triggers'), + restoreTimeout: $config->get('db-tools.restore_timeout', 1800), + storageDirectory: $config->get('db-tools.storage_directory', $app->storagePath('db_tools')), + storageFilenameStrategy: $config->get('db-tools.storage_filename_strategy', 'default'), + ); + + $connectionConfigs = []; + foreach ($config->get('db-tools.connections', []) as $name => $data) { + $connectionConfigs[$name] = new Configuration( + backupBinary: $data['backup_binary'] ?? null, + backupExcludedTables: $data['backup_excluded_tables'] ?? null, + backupExpirationAge: $data['backup_expiration_age'] ?? null, + backupOptions: $data['backup_options'] ?? null, + backupTimeout: $data['backup_timeout'] ?? null, + restoreBinary: $data['restore_binary'] ?? null, + restoreOptions: $data['restore_options'] ?? null, + restoreTimeout: $data['restore_timeout'] ?? null, + parent: $defaultConfig, + storageDirectory: $data['storage_directory'] ?? null, + storageFilenameStrategy: $data['storage_filename_strategy'] ?? null, + ); + } + + return new ConfigurationRegistry( + $defaultConfig, + $connectionConfigs, + $config->get('database.default') + ); + }); + + $this->app->singleton(Storage::class, function (Application $app) { + /** @var ConfigurationRegistry $configRegistry */ + $configRegistry = $app->make(ConfigurationRegistry::class); + + $connections = $configRegistry->getConnectionConfigAll(); + if (empty($connections)) { + $connections[$configRegistry->getDefaultConnection()] = $configRegistry->getDefaultConfig(); + } + + // Register filename strategies. + $strategies = []; + foreach ($connections as $connectionName => $connection) { + $strategyId = $connection->getStorageFilenameStrategy(); + + if ($strategyId === null || $strategyId === 'default' || $strategyId === 'datetime') { + $strategy = new DefaultFilenameStrategy(); + } elseif ($app->bound($strategyId)) { + $strategy = $app->make($strategyId); + } elseif (\class_exists($strategyId)) { + if (!\is_subclass_of($strategyId, FilenameStrategyInterface::class)) { + throw new \InvalidArgumentException(\sprintf( + '"db-tools.connections.%s.filename_strategy": class "%s" does not implement "%s"', + $connectionName, + $strategyId, + FilenameStrategyInterface::class + )); + } + $strategy = $app->make($strategyId); + } else { + throw new \InvalidArgumentException(\sprintf( + '"db-tools.connections.%s.filename_strategy": class or service "%s" does not exist or is not registered in container', + $connectionName, + $strategyId + )); + } + + $strategies[$connectionName] = $strategy; + } + + return new Storage($configRegistry, $strategies); + }); + + $this->app->resolving( + AnonymizatorFactory::class, + function (AnonymizatorFactory $factory, Application $app): void { + /** @var Repository $config */ + $config = $app->make('config'); + + foreach ($config->get('db-tools.anonymization_files', []) as $connectionName => $file) { + // 0 is not a good index for extension, this fails for false and 0. + if (!($pos = \strrpos($file, '.'))) { + throw new ConfigurationException(\sprintf( + "File extension cannot be guessed for \"%s\" file path.", $file + )); + } + + $ext = \substr($file, $pos + 1); + $loader = match ($ext) { + 'php' => new PhpFileLoader($file, $connectionName), + 'yml', 'yaml' => new YamlLoader($file, $connectionName), + default => throw new ConfigurationException(\sprintf( + "File extension \"%s\" is unsupported (given path: \"%s\").", $ext, $file + )), + }; + + $factory->addConfigurationLoader($loader); + } + + foreach ($config->get('db-tools.anonymization', []) as $connectionName => $array) { + $factory->addConfigurationLoader(new ArrayLoader($array, $connectionName)); + } + } + ); + + $this->app + ->when(AnonymizerRegistry::class) + ->needs('$paths') + ->give(function (Application $app) { + // Validate user-given anonymizer paths. + $anonymizerPaths = $app->make('config')->get('db-tools.anonymizer_paths'); + foreach ($anonymizerPaths as $path) { + if (!\is_dir($path)) { + throw new \InvalidArgumentException(\sprintf( + '"db_tools.anonymizer_paths": path "%s" does not exist', $path + )); + } + } + // Set the default anonymizer directory only if the folder + // exists in order to avoid "directory does not exist" errors. + $defaultDirectory = $app->basePath('app/Anonymizer'); + if (\is_dir($defaultDirectory)) { + $anonymizerPaths[] = $defaultDirectory; + } + }) + ; + + /*$this->app->singleton(BackupperFactory::class, function (Application $app) { + return new BackupperFactory( + $app->make(DatabaseSessionRegistry::class), + $app->make(ConfigurationRegistry::class), + // Logger + ); + });*/ + + /*$this->app->singleton(RestorerFactory::class, function (Application $app) { + return new RestorerFactory( + $app->make(DatabaseSessionRegistry::class), + $app->make(ConfigurationRegistry::class), + // Logger + ); + });*/ + + /*$this->app->singleton(StatsProviderFactory::class, function (Application $app) { + return new StatsProviderFactory($app->make(DatabaseSessionRegistry::class)); + });*/ + + /*$this->app + ->when([ + AnonymizatorFactory::class, + BackupperFactory::class, + RestorerFactory::class, + StatsProviderFactory::class + ]) + ->needs(LoggerInterface::class) + ->give(fn (Application $app) => $app->make('log')) + ;*/ + + $this->app + ->when([ + AnonymizeCommand::class, + BackupCommand::class, + RestoreCommand::class, + ]) + ->needs('$connectionName') + ->giveConfig('database.default') + ; + $this->app + ->when([ + CheckCommand::class, + CleanCommand::class, + StatsCommand::class, + ]) + ->needs('$defaultConnectionName') + ->giveConfig('database.default') + ; + } + + /** + * Bootstrap any package services. + */ + public function boot(): void + { + AboutCommand::add('DbToolsBundle', fn () => ['Version' => '2.0.0']); + + $this->publishes([ + __DIR__ . '/Resources/config/db-tools.php' => $this->app->configPath('db-tools.php'), + ]); + + if ($this->app->runningInConsole()) { + $this->commands([ + AnonymizeCommand::class, + AnonymizerListCommand::class, + BackupCommand::class, + CheckCommand::class, + CleanCommand::class, + ConfigDumpCommand::class, + RestoreCommand::class, + StatsCommand::class, + ]); + } + } +} diff --git a/src/Bridge/Laravel/LaravelDatabaseSessionRegistry.php b/src/Bridge/Laravel/LaravelDatabaseSessionRegistry.php new file mode 100644 index 00000000..92c245a0 --- /dev/null +++ b/src/Bridge/Laravel/LaravelDatabaseSessionRegistry.php @@ -0,0 +1,84 @@ +databaseManager->getConnections()); + } + + #[\Override] + public function getDefaultConnectionName(): string + { + return $this->databaseManager->getDefaultConnection(); + } + + #[\Override] + public function getConnectionDsn(string $name): Dsn + { + $params = \array_filter($this->databaseManager->connection($name)->getConfig()); + + if (empty($params['driver'])) { + throw new \DomainException("Database connection 'driver' parameter is missing."); + } + + $vendor = $params['driver']; + $host = $params['host'] ?? null; + $filename = $params['path'] ?? $params['unix_socket'] ?? null; + $database = $params['database'] ?? null; + $user = $params['username'] ?? null; + $password = $params['password'] ?? null; + $port = (int) $params['port'] ?? null; + + unset( + $params['database'], + $params['driver'], + $params['host'], + $params['password'], + $params['path'], + $params['port'], + $params['unix_socket'], + $params['user'], + ); + + return new Dsn( + database: $database, + filename: $filename, + host: $host, + password: $password, + port: $port, + query: $params, + user: $user, + vendor: $vendor, + ); + } + + #[\Override] + public function getDatabaseSession(string $name): DatabaseSession + { + /** @var Connection $doctrineConnection */ + $doctrineConnection = $this + ->databaseManager + ->connection($name) + ->getDoctrineConnection() + ; + + return new DoctrineBridge($doctrineConnection); + } +} diff --git a/src/Bridge/Laravel/Resources/config/db-tools.php b/src/Bridge/Laravel/Resources/config/db-tools.php new file mode 100644 index 00000000..8154e0c1 --- /dev/null +++ b/src/Bridge/Laravel/Resources/config/db-tools.php @@ -0,0 +1,218 @@ + env('DBTOOLS_STORAGE_DIRECTORY', storage_path('db_tools')), + + /* + | ------------------------------------------------------------------------- + | Storage filename strategy + | ------------------------------------------------------------------------- + | + | Filename strategies for backups storage. You may specify one strategy for + | each database connection. Keys are connection names, values are strategy + | names. + | + | A strategy name can be either the class name (FQN) of a custom strategy + | implementation, or its potential "little name" or "identifier" if you + | registered it into the container without using directly its class name. + | + | "default", null, or omitting the connection are equivalent and involve + | the use of the default implementation. + | + | To summarize, allowed values are: + | - "default": alias of "datetime". + | - "datetime": implementation producing names formatted as such: + | /YYYY/MM/-.. + | - CLASS_NAME: class name of a custom strategy implementation. + | - SERVICE_ID: identifier of a custom strategy implementation registered + | as a service into the container. + | + */ + + 'storage_filename_strategy' => env('DBTOOLS_STORAGE_FILENAME_STRATEGY', 'default'), + + /* + | ------------------------------------------------------------------------- + | Backup file expiration + | ------------------------------------------------------------------------- + | + | Indicate when old backups can be considered obsolete. + | + | Use relative date/time formats: + | https://www.php.net/manual/en/datetime.formats.relative.php + | + */ + + 'backup_expiration_age' => env('DBTOOLS_BACKUP_EXPIRATION_AGE', '3 months ago'), + + /* + | ------------------------------------------------------------------------- + | Backup excluded tables + | ------------------------------------------------------------------------- + | + | List here database tables you don't want to include in your backups. + | + | If you have more than one database connection, it is strongly advised + | to configure this for each connection instead. + | + */ + + 'backup_excluded_tables' => env('DBTOOLS_BACKUP_EXCLUDED_TABLES', []), + + /* + | ------------------------------------------------------------------------- + | Process timeouts + | ------------------------------------------------------------------------- + | + | Default timeouts for backup and restoration processes. + | + */ + + 'backup_timeout' => env('DBTOOLS_BACKUP_TIMEOUT', 600), + 'restore_timeout' => env('DBTOOLS_RESTORE_TIMEOUT', 1800), + + /* + | ------------------------------------------------------------------------- + | Binaries & options + | ------------------------------------------------------------------------- + | + | Specify here paths to backup and restoration binaries as well as their + | respective command line options. + | + | Warning: this will apply to all connections disregarding their database + | vendor. If you have more than one connection and if they use different + | database vendors or versions, please configure those for each connection + | instead. + | + | Default values depends upon vendor and are documented at + | https://dbtoolsbundle.readthedocs.io/en/stable/configuration.html + | + */ + + 'backup_binary' => env('DBTOOLS_BACKUP_BINARY', '/usr/bin/pg_dump'), + 'backup_options' => env('DBTOOLS_BACKUP_OPTIONS', '-Z 5 --lock-wait-timeout=120'), + 'restore_binary' => env('DBTOOLS_RESTORE_BINARY', '/usr/bin/pg_restore'), + 'restore_options' => env('DBTOOLS_RESTORE_OPTIONS', '-j 2 --clean --if-exists --disable-triggers'), + + /* + | ------------------------------------------------------------------------- + | Connection specific parameters + | ------------------------------------------------------------------------- + | + | For advanced usage, you may also override any parameter for each database + | connection. All parameters defined above are allowed. Keys are connection + | names. + | + | Example: + | + | 'connections' => [ + | 'specific_connection' => [ + | 'backup_binary' => '/usr/local/bin/vendor-one-dump', + | 'backup_excluded_tables' => ['table_one', 'table_two'], + | 'backup_expiration_age' => '1 month ago', + | 'backup_options' => '--no-table-lock', + | 'backup_timeout' => 2000, + | 'restore_binary' => '/usr/local/bin/vendor-one-restore', + | 'restore_options' => '--disable-triggers --other-option', + | 'restore_timeout' => 5000, + | 'storage_directory' => '/path/to/storage', + | 'storage_filename_strategy' => 'datetime', + | ], + | ], + | + */ + + 'connections' => [], + + /* + | ------------------------------------------------------------------------- + | Anonymizer folders + | ------------------------------------------------------------------------- + | + | Update this configuration if you want to look for additional anonymizers + | in custom folders. + | + | Keep in mind that both following folders are automatically registered, + | don't repeat them: + | - app/Anonymization/Anonymizer + | - vendor/makinacorpus/db-tools-bundle/src/Anonymization/Anonymizer/Core + | + */ + + 'anonymizer_paths' => [], + + /* + | ------------------------------------------------------------------------- + | Anonymization configuration + | ------------------------------------------------------------------------- + | + | For simple needs, you may simply write the anonymization configuration + | here. Keys are connection names, values are structures identical to what + | you may find in the "anonymizations.sample.yaml" example. + | + | Example: + | + | 'anonymization' => [ + | 'specific_connection' => [ + | 'table1' => [ + | 'column1' => [ + | 'anonymizer' => 'anonymizer_name', + | // Anonymizer specific options... + | ], + | 'column2' => [ + | // ... + | ], + | ], + | 'table2' => [ + | // ... + | ], + | ], + | ], + | + */ + + 'anonymization' => [], + + /* + | ------------------------------------------------------------------------- + | Anonymization configuration files + | ------------------------------------------------------------------------- + | + | You can, for organisation purpose, delegate anonymization configuration + | into extra configuration files, and simply reference them here. + | + | File paths can be either relative or absolute. Relative paths are + | relative to the workdir option if specified, or from this configuration + | file directory otherwise. + | + | You can use PHP or YAML files to configure your anonymization. + | + | See the "config/anonymizations.sample.yaml" file included in the + | makinacorpus/db-tools-bundle package as an example. + | + | Example: + | + | 'anonymization_files' => [ + | 'connection_one' => database_path('anonymization/connection_one.php'), + | 'connection_two' => database_path('anonymization/connection_two.php'), + | ], + | + | If you have only one connection, you can adopt the following syntax: + | + | 'anonymization_files' => database_path('anonymization.php'), + | + */ + + 'anonymization_files' => [], +];