diff --git a/composer.json b/composer.json index 0e10547c..daff1cd7 100644 --- a/composer.json +++ b/composer.json @@ -13,14 +13,15 @@ "prefer-stable": true, "minimum-stability": "dev", "require": { - "pimcore/compatibility-bridge-v10": "^1.0", "php": "~8.0.0 || ~8.1.0 || ~8.2.0 || ~8.3.0", "ext-fileinfo": "*", "ext-json": "*", + "doctrine/dbal": "^2.12 || ^3.5", "dragonmantank/cron-expression": "^3.1", "league/flysystem-sftp-v3": "^3.0", "nesbot/carbon": "^2.27", "phpoffice/phpspreadsheet": "^1.24 || ^2.2", + "pimcore/compatibility-bridge-v10": "^1.0", "pimcore/data-hub": "^1.6", "pimcore/pimcore": "^10.6 || ^11.0", "symfony/mime": "^5.2 || ^6.2", diff --git a/doc/03_Configuration/01_Data_Sources.md b/doc/03_Configuration/01_Data_Sources.md index 6440830e..26276c11 100644 --- a/doc/03_Configuration/01_Data_Sources.md +++ b/doc/03_Configuration/01_Data_Sources.md @@ -52,3 +52,42 @@ The URL for the endpoint is: `http(s)://>/pimcore-datahub-import/ + +![Data Source SQL](../img/datasource_sql.png) + +Loads data from a defined doctrine connection. + +The SQL Data Loader uses [DBAL](https://www.doctrine-project.org/projects/dbal.html) to allow data to be +loaded from a SQL source. Connections to any database supported by DBAL will work provided they are +configured correctly inside of `database.yaml`. (Database configuration can be placed in any valid +Symfony config file, provided its in the correct format as can be seen in `database.yaml`). + +Example connection: +```yaml +doctrine: + dbal: + connections: + new_connection: + host: db + port: '3306' + user: sample_user + password: sample_password + dbname: sample_dbname + driver: any_supported_by_doctrine +``` + +For different drivers some additional configuration could be needed. + +##### Configuration Options: +- **Connection**: Connection from which data will be loaded +- **SELECT**: Valid SQL `SELECT` +- **FROM**: Valid SQL `FROM` +- **WHERE**: Valid SQL `WHERE` +- **GROUP BY**: Valid SQL `GROUP BY` + +Ensure to select **SQL** under File Format! \ No newline at end of file diff --git a/doc/img/datasource_sql.png b/doc/img/datasource_sql.png new file mode 100644 index 00000000..70ce05af Binary files /dev/null and b/doc/img/datasource_sql.png differ diff --git a/src/Controller/ConnectionController.php b/src/Controller/ConnectionController.php new file mode 100644 index 00000000..ac65fd53 --- /dev/null +++ b/src/Controller/ConnectionController.php @@ -0,0 +1,50 @@ +getParameter('doctrine.connections'); + + if (!is_array($connections)) { + throw new Exception('Doctrine connection not returned as array'); + } + + $mappedConnections = array_map(fn ($key, $value): array => [ + 'name' => $key, + 'value' => $value + ], array_keys($connections), $connections); + + return $this->json($mappedConnections); + } +} diff --git a/src/DataSource/Interpreter/SqlFileInterpreter.php b/src/DataSource/Interpreter/SqlFileInterpreter.php new file mode 100644 index 00000000..f5aec271 --- /dev/null +++ b/src/DataSource/Interpreter/SqlFileInterpreter.php @@ -0,0 +1,22 @@ +setUpConnection(); + $this->setUpImportFilePath(); + + $queryBuilder = $this->databaseConnection->createQueryBuilder(); + $queryBuilder->select($this->select) + ->from($this->from); + + if (!empty($this->where)) { + $queryBuilder->where($this->where); + } + + if (!empty($this->groupBy)) { + $queryBuilder->groupBy($this->groupBy); + } + + $result = $queryBuilder->execute()->fetchAllAssociative(); + + $filesystemLocal = new Filesystem(new LocalFilesystemAdapter('/')); + $stream = fopen('php://temp', 'r+'); + $resultAsJson = json_encode($result); + + fwrite($stream, $resultAsJson); + rewind($stream); + + $filesystemLocal->writeStream($this->importFilePath, $stream); + + return $this->importFilePath; + } + + public function cleanup(): void + { + $this->databaseConnection->close(); + + unlink($this->importFilePath); + } + + /** + * @throws InvalidConfigurationException + */ + public function setSettings(array $settings): void + { + if (empty($settings['connection'])) { + throw new InvalidConfigurationException('Empty connection.'); + } + $this->connection = $settings['connection']; + + if (empty($settings['select'])) { + throw new InvalidConfigurationException('Empty select.'); + } + $this->select = $settings['select']; + + if (empty($settings['from'])) { + throw new InvalidConfigurationException('Empty from.'); + } + $this->from = $settings['from']; + + $this->where = $settings['where']; + $this->groupBy = $settings['groupBy']; + } + + /** + * @throws InvalidConfigurationException + */ + private function setUpConnection(): void + { + $container = Pimcore::getContainer(); + $databaseConnection = null; + + if ($container instanceof Component\DependencyInjection\ContainerInterface) { + $databaseConnection = $container->get($this->connection); + } + + if (!$databaseConnection instanceof Connection) { + throw new InvalidConfigurationException('Connection not found.'); + } + + $this->databaseConnection = $databaseConnection; + } + + private function setUpImportFilePath(): void + { + $folder = PIMCORE_PRIVATE_VAR . '/tmp/datahub/dataimporter/sql-loader/'; + $this->filesystem->mkdir($folder, 0775); + + $this->importFilePath = $folder . uniqid('sql-import-'); + } +} diff --git a/src/PimcoreDataImporterBundle.php b/src/PimcoreDataImporterBundle.php index 76db64d1..0998dff5 100644 --- a/src/PimcoreDataImporterBundle.php +++ b/src/PimcoreDataImporterBundle.php @@ -70,10 +70,12 @@ public function getJsPaths(): array '/bundles/pimcoredataimporter/js/pimcore/configuration/components/loader/asset.js', '/bundles/pimcoredataimporter/js/pimcore/configuration/components/loader/upload.js', '/bundles/pimcoredataimporter/js/pimcore/configuration/components/loader/push.js', + '/bundles/pimcoredataimporter/js/pimcore/configuration/components/loader/sql.js', '/bundles/pimcoredataimporter/js/pimcore/configuration/components/interpreter/csv.js', '/bundles/pimcoredataimporter/js/pimcore/configuration/components/interpreter/json.js', '/bundles/pimcoredataimporter/js/pimcore/configuration/components/interpreter/xlsx.js', '/bundles/pimcoredataimporter/js/pimcore/configuration/components/interpreter/xml.js', + '/bundles/pimcoredataimporter/js/pimcore/configuration/components/interpreter/sql.js', '/bundles/pimcoredataimporter/js/pimcore/configuration/components/cleanup/unpublish.js', '/bundles/pimcoredataimporter/js/pimcore/configuration/components/cleanup/delete.js', '/bundles/pimcoredataimporter/js/pimcore/configuration/components/importSettings.js', diff --git a/src/Resources/config/pimcore/routing.yml b/src/Resources/config/pimcore/routing.yml index f976c452..e73fef32 100644 --- a/src/Resources/config/pimcore/routing.yml +++ b/src/Resources/config/pimcore/routing.yml @@ -4,6 +4,12 @@ _data_hub_data_importer_data_object_admin: options: expose: true +_data_hub_data_importer_connections: + resource: "@PimcoreDataImporterBundle/Controller/ConnectionController.php" + type: annotation + options: + expose: true + _data_hub_data_importer_data_object_push: resource: "@PimcoreDataImporterBundle/Controller/PushImportController.php" type: annotation diff --git a/src/Resources/config/services.yml b/src/Resources/config/services.yml index 10e664a4..8c887b5e 100644 --- a/src/Resources/config/services.yml +++ b/src/Resources/config/services.yml @@ -59,6 +59,10 @@ services: tags: - { name: "pimcore.datahub.data_importer.loader", type: "push" } + Pimcore\Bundle\DataImporterBundle\DataSource\Loader\SqlLoader: + tags: + - { name: "pimcore.datahub.data_importer.loader", type: "sql" } + Pimcore\Bundle\DataImporterBundle\DataSource\Interpreter\DeltaChecker\DeltaChecker: ~ Pimcore\Bundle\DataImporterBundle\DataSource\Interpreter\InterpreterFactory: ~ @@ -91,6 +95,13 @@ services: - { name: monolog.logger, channel: 'DATA-IMPORTER' } - { name: "pimcore.datahub.data_importer.interpreter", type: "xml" } + Pimcore\Bundle\DataImporterBundle\DataSource\Interpreter\SqlFileInterpreter: + calls: + - [ setLogger, [ '@logger' ] ] + tags: + - { name: monolog.logger, channel: 'DATA-IMPORTER' } + - { name: "pimcore.datahub.data_importer.interpreter", type: "sql" } + Pimcore\Bundle\DataImporterBundle\Cleanup\CleanupStrategyFactory: ~ Pimcore\Bundle\DataImporterBundle\Cleanup\DeleteStrategy: diff --git a/src/Resources/public/js/pimcore/configuration/components/interpreter/sql.js b/src/Resources/public/js/pimcore/configuration/components/interpreter/sql.js new file mode 100644 index 00000000..49e623a5 --- /dev/null +++ b/src/Resources/public/js/pimcore/configuration/components/interpreter/sql.js @@ -0,0 +1,36 @@ +/** + * Pimcore + * + * This source file is available under two different licenses: + * - GNU General Public License version 3 (GPLv3) + * - Pimcore Commercial License (PCL) + * Full copyright and license information is available in + * LICENSE.md which is distributed with this source code. + * + * @copyright Copyright (c) Pimcore GmbH (http://www.pimcore.org) + * @license http://www.pimcore.org/license GPLv3 and PCL + */ + +pimcore.registerNS("pimcore.plugin.pimcoreDataImporterBundle.configuration.components.interpreter.sql"); +pimcore.plugin.pimcoreDataImporterBundle.configuration.components.interpreter.sql = Class.create(pimcore.plugin.pimcoreDataImporterBundle.configuration.components.abstractOptionType, { + + type: 'sql', + + buildSettingsForm: function() { + + if(!this.form) { + this.form = Ext.create('DataHub.DataImporter.StructuredValueForm', { + defaults: { + labelWidth: 200, + width: 600 + }, + border: false, + items: [ + ] + }); + } + + return this.form; + } + +}); \ No newline at end of file diff --git a/src/Resources/public/js/pimcore/configuration/components/loader/sql.js b/src/Resources/public/js/pimcore/configuration/components/loader/sql.js new file mode 100644 index 00000000..63ae901a --- /dev/null +++ b/src/Resources/public/js/pimcore/configuration/components/loader/sql.js @@ -0,0 +1,104 @@ +/** + * Pimcore + * + * This source file is available under two different licenses: + * - GNU General Public License version 3 (GPLv3) + * - Pimcore Commercial License (PCL) + * Full copyright and license information is available in + * LICENSE.md which is distributed with this source code. + * + * @copyright Copyright (c) Pimcore GmbH (http://www.pimcore.org) + * @license http://www.pimcore.org/license GPLv3 and PCL + */ + +pimcore.registerNS("pimcore.plugin.pimcoreDataImporterBundle.configuration.components.loader.sql"); +pimcore.plugin.pimcoreDataImporterBundle.configuration.components.loader.sql = Class.create(pimcore.plugin.pimcoreDataImporterBundle.configuration.components.abstractOptionType, { + + type: 'sql', + + buildSettingsForm: function() { + + if(!this.form) { + const dataStore = Ext.create('Ext.data.Store', { + autoLoad: true, + proxy: { + type: 'ajax', + url: Routing.generate('pimcore_dataimporter_connections'), + reader: { + type: 'json' + } + } + }); + + this.form = Ext.create('DataHub.DataImporter.StructuredValueForm', { + defaults: { + labelWidth: 200, + width: 600 + }, + border: false, + items: [ + { + xtype: 'combobox', + fieldLabel: t('plugin_pimcore_datahub_data_importer_configpanel_sql_connection'), + name: this.dataNamePrefix + 'connection', + value: this.data.connection, + allowBlank: false, + msgTarget: 'under', + displayField: 'name', + valueField: 'value', + store: dataStore, + }, + { + xtype: 'textarea', + fieldLabel: "SELECT
(eg. a,b,c)*", + name: this.dataNamePrefix + 'select', + value: this.data.select, + msgTarget: 'under', + width: 900, + height: 200, + grow: true, + growMax: 400, + allowBlank: false, + }, + { + xtype: 'textarea', + fieldLabel: "FROM
(eg. d INNER JOIN e ON c.a = e.b)*", + name: this.dataNamePrefix + 'from', + value: this.data.from, + msgTarget: 'under', + width: 900, + height: 200, + grow: true, + growMax: 400, + allowBlank: false, + }, + { + xtype: 'textarea', + fieldLabel: "WHERE
(eg. c = 'some_value')", + name: this.dataNamePrefix + 'where', + value: this.data.where, + msgTarget: 'under', + width: 900, + height: 200, + grow: true, + growMax: 400, + }, + { + xtype: 'textarea', + fieldLabel: "GROUP BY
(eg. b, c )", + name: this.dataNamePrefix + 'groupBy', + value: this.data.groupBy, + msgTarget: 'under', + width: 900, + height: 200, + grow: true, + growMax: 400, + }, + ] + }); + } + + return this.form; + } + +}); \ No newline at end of file diff --git a/src/Resources/translations/admin.en.yml b/src/Resources/translations/admin.en.yml index 700bf11f..83ea71db 100644 --- a/src/Resources/translations/admin.en.yml +++ b/src/Resources/translations/admin.en.yml @@ -206,3 +206,5 @@ plugin_pimcore_datahub_data_importer_configpanel_execution_date_format: Date &am plugin_pimcore_datahub_data_importer_configpanel_logs: Import Logs plugin_pimcore_datahub_data_importer_configpanel_mtm_relation_type_error: Type not supported for Many To Many Relation assignment. plugin_pimcore_datahub_data_importer_configpanel_mtm_relation_type: Make sure transformation results in a 'dataObjectArray' or an 'assetArray'. +plugin_pimcore_datahub_data_importer_configpanel_type_sql: SQL +plugin_pimcore_datahub_data_importer_configpanel_sql_connection: Connection \ No newline at end of file