From 3afc4a139761fa7c366de3bd250772bd1a4e77cb Mon Sep 17 00:00:00 2001 From: Yurii Hryhoriev Date: Thu, 8 Dec 2016 12:44:56 +0200 Subject: [PATCH 01/17] MAGETWO-62034: [DB-PPD] Data migration fails on Customer Step --- src/Migration/Step/Customer/Data.php | 11 +- src/Migration/Step/Customer/Helper.php | 261 +++++++----- src/Migration/Step/Customer/Integrity.php | 21 +- .../Migration/Step/Customer/HelperTest.php | 374 +++++++++++------- 4 files changed, 414 insertions(+), 253 deletions(-) diff --git a/src/Migration/Step/Customer/Data.php b/src/Migration/Step/Customer/Data.php index 54abe58c3..3a77d3a06 100644 --- a/src/Migration/Step/Customer/Data.php +++ b/src/Migration/Step/Customer/Data.php @@ -132,7 +132,7 @@ public function perform() ); $recordTransformer->init(); - $attributeType = $this->helper->getAttributeType($sourceDocName); + $entityTypeCode = $this->helper->getEntityTypeCodeByDocumentName($sourceDocName); $pageNumber = 0; $this->logger->debug('migrating', ['table' => $sourceDocName]); @@ -146,7 +146,7 @@ public function perform() $destinationRecords = $destDocument->getRecords(); foreach ($bulk as $recordData) { $this->source->setLastLoadedRecord($sourceDocName, $recordData); - if ($this->helper->isSkipRecord($attributeType, $sourceDocName, $recordData)) { + if ($this->helper->isSkipRecord($entityTypeCode, $sourceDocName, $recordData)) { continue; } /** @var Record $record */ @@ -159,7 +159,12 @@ public function perform() $this->progress->advance(LogManager::LOG_LEVEL_DEBUG); - $this->helper->updateAttributeData($attributeType, $sourceDocName, $destinationRecords); + $this->helper->updateAttributeData( + $entityTypeCode, + $sourceDocName, + $destinationName, + $destinationRecords + ); $this->destination->saveRecords($destinationName, $destinationRecords); } diff --git a/src/Migration/Step/Customer/Helper.php b/src/Migration/Step/Customer/Helper.php index 35c0910d3..9cd3d094b 100644 --- a/src/Migration/Step/Customer/Helper.php +++ b/src/Migration/Step/Customer/Helper.php @@ -45,17 +45,17 @@ class Helper protected $readerAttributes; /** - * @var [] + * @var array */ protected $eavAttributes; /** - * @var [] + * @var array */ protected $skipAttributes; /** - * @var [] + * @var array */ protected $sourceDocuments; @@ -87,32 +87,38 @@ public function __construct( } /** - * @param string $document + * Retrieves entity type code by document name + * + * @param string $sourceDocName * @return string|null * @throws \Migration\Exception */ - public function getAttributeType($document) + public function getEntityTypeCodeByDocumentName($sourceDocName) { if (empty($this->documentAttributeTypes)) { - $entities = array_keys($this->readerGroups->getGroup('eav_entities')); - foreach ($entities as $entity) { - $documents = $this->readerGroups->getGroup($entity); - foreach ($documents as $item => $key) { - $this->documentAttributeTypes[$item] = $entity; - $this->initEavEntity($entity, $item, $key); + $entityTypeCodes = array_keys($this->readerGroups->getGroup('eav_entities')); + foreach ($entityTypeCodes as $entityTypeCode) { + $documents = $this->readerGroups->getGroup($entityTypeCode); + $documents = array_keys($documents); + foreach ($documents as $documentName) { + $this->documentAttributeTypes[$documentName] = $entityTypeCode; } } } - return isset($this->documentAttributeTypes[$document]) ? $this->documentAttributeTypes[$document] : null; + return isset($this->documentAttributeTypes[$sourceDocName]) ? + $this->documentAttributeTypes[$sourceDocName] : + null; } /** - * @param string $attributeType + * Retrieves whether record should be skipped in moving from source to destination + * + * @param string $entityTypeCode * @param string $sourceDocName * @param [] $recordData * @return bool */ - public function isSkipRecord($attributeType, $sourceDocName, $recordData) + public function isSkipRecord($entityTypeCode, $sourceDocName, $recordData) { if (!isset($this->sourceDocuments[$sourceDocName]) || $this->sourceDocuments[$sourceDocName] != 'value_id' @@ -120,17 +126,79 @@ public function isSkipRecord($attributeType, $sourceDocName, $recordData) ) { return false; } - return isset($this->skipAttributes[$attributeType][$recordData['attribute_id']]); + + $skippedRecords = $this->getSkippedAttributes(); + return isset($skippedRecords[$entityTypeCode][$recordData['attribute_id']]); + } + + /** + * Retrieves array with attributes which should be skipped + * + * Sample of returning array: + * + * [ + * 'customer' => [ + * 3 => 'created_in', + * 4 => 'prefix', + * 5 => 'firstname' + * ], + * 'customer_address' => [ + * 26 => 'city', + * 24 => 'company', + * 27 => 'country_id' + * ] + * ] + * + * @return array + * @throws \Migration\Exception + */ + public function getSkippedAttributes() + { + if ($this->skipAttributes === null) { + $this->skipAttributes = []; + $entityTypeCodes = array_keys($this->readerGroups->getGroup('eav_entities')); + foreach ($entityTypeCodes as $entityTypeCode) { + $documents = $this->readerGroups->getGroup($entityTypeCode); + foreach ($documents as $documentName => $key) { + if ($key == 'entity_id') { + foreach (array_keys($this->readerAttributes->getGroup($documentName)) as $attributeCode) { + $eavAttributes = $this->getAttributesData($entityTypeCode); + if (empty($eavAttributes)) { + throw new \Migration\Exception( + sprintf('Attribute type %s does not exist', $entityTypeCode) + ); + } elseif (!isset($eavAttributes[$attributeCode])) { + throw new \Migration\Exception( + sprintf( + 'Attribute %s does not exist in the type %s', + $attributeCode, + $entityTypeCode + ) + ); + } + $attributeId = $eavAttributes[$attributeCode]['attribute_id']; + $this->skipAttributes[$entityTypeCode][$attributeId] = $attributeCode; + } + } + } + } + } + return $this->skipAttributes; } /** - * @param string $attributeType + * @param string $entityTypeCode * @param string $sourceDocName + * @param string $destinationDocName * @param Record\Collection $destinationRecords * @return void */ - public function updateAttributeData($attributeType, $sourceDocName, Record\Collection $destinationRecords) - { + public function updateAttributeData( + $entityTypeCode, + $sourceDocName, + $destinationDocName, + Record\Collection $destinationRecords + ) { if (!isset($this->sourceDocuments[$sourceDocName]) || $this->sourceDocuments[$sourceDocName] != 'entity_id') { return; } @@ -143,11 +211,13 @@ public function updateAttributeData($attributeType, $sourceDocName, Record\Colle $attributeIdsByType = []; $attributeCodesById = []; $attributeCodes = array_keys($this->readerAttributes->getGroup($sourceDocName)); - foreach ($attributeCodes as $attribute) { - if (isset($this->eavAttributes[$attributeType][$attribute])) { - $attributeId = $this->eavAttributes[$attributeType][$attribute]['attribute_id']; - $attributeIdsByType[$this->eavAttributes[$attributeType][$attribute]['backend_type']][] = $attributeId; - $attributeCodesById[$attributeId] = $attribute; + foreach ($attributeCodes as $attributeCode) { + $eavAttributes = $this->getAttributesData($entityTypeCode); + if (is_array($eavAttributes) && isset($eavAttributes[$attributeCode])) { + $attributeId = $eavAttributes[$attributeCode]['attribute_id']; + $attributeBackendType = $eavAttributes[$attributeCode]['backend_type']; + $attributeIdsByType[$attributeBackendType][] = $attributeId; + $attributeCodesById[$attributeId] = $attributeCode; } } @@ -171,29 +241,62 @@ public function updateAttributeData($attributeType, $sourceDocName, Record\Colle $attributeCodesById ); - $this->setAttributeData($destinationRecords, $recordAttributesData, $attributeCodes); + $fieldsWithDefaults = $this->getFieldsWithDefaultValues($attributeCodes, $destinationDocName); + $this->setAttributeData($destinationRecords, $recordAttributesData, $fieldsWithDefaults); } + /** + * Retrieves destination document fields with valid default values + * + * @param array $attributeCodes + * @param string $destinationDocName + * @return array + * @throws \Migration\Exception + */ + protected function getFieldsWithDefaultValues($attributeCodes, $destinationDocName) + { + /** @var Mysql $adapter */ + $adapter = $this->destination->getAdapter(); + $structure = $adapter->getDocumentStructure($destinationDocName); + + $fields = []; + foreach ($attributeCodes as $attributeCode) { + if (!isset($structure[$attributeCode])) { + throw new \Migration\Exception( + sprintf('Destination document %s does not have attribute %s', $destinationDocName, $attributeCode) + ); + } + $fields[$attributeCode] = + $structure[$attributeCode]['DEFAULT'] === null && $structure[$attributeCode]['NULLABLE'] === false ? + '' : + null; + } + return $fields; + } + + /** + * @param Record\Collection $destinationRecords + * @param array $recordAttributesData + * @param array $fieldsWithDefaults + * @return void + */ public function setAttributeData( Record\Collection $destinationRecords, array $recordAttributesData, - array $attributeCodes + array $fieldsWithDefaults ) { /** @var Record $record */ foreach ($destinationRecords as $record) { + $recordEntityData = []; if (isset($recordAttributesData[$record->getValue('entity_id')])) { $recordEntityData = $recordAttributesData[$record->getValue('entity_id')]; if ($this->configReader->getOption(self::UPGRADE_CUSTOMER_PASSWORD_HASH)) { $recordEntityData = $this->upgradeCustomerHash($recordEntityData); } - $data = $record->getData(); - $data = array_merge( - array_fill_keys($attributeCodes, null), - $data, - $recordEntityData - ); - $record->setData($data); } + $data = $record->getData(); + $data = array_merge($fieldsWithDefaults, $data, $recordEntityData); + $record->setData($data); } } @@ -221,7 +324,8 @@ public function getSourceRecordsCount($document) if ($this->sourceDocuments[$document] == 'entity_id') { return $this->source->getRecordsCount($document); } - $attributeType = $this->getAttributeType($document); + $attributeType = $this->getEntityTypeCodeByDocumentName($document); + $skipAttributes = $this->getSkippedAttributes(); /** @var Mysql $adapter */ $adapter = $this->source->getAdapter(); @@ -232,75 +336,48 @@ public function getSourceRecordsCount($document) ], 'COUNT(*)' ) - ->where('et.attribute_id NOT IN (?)', array_keys($this->skipAttributes[$attributeType])); + ->where('et.attribute_id NOT IN (?)', array_keys($skipAttributes[$attributeType])); $count = $query->getAdapter()->fetchOne($query); return $count; } /** - * @param string $attributeType - * @param string $document - * @param string $key - * @return void - * @throws \Migration\Exception - */ - protected function initEavEntity($attributeType, $document, $key) - { - if ($key != 'entity_id') { - return; - } - $this->initEavAttributes($attributeType); - foreach (array_keys($this->readerAttributes->getGroup($document)) as $attribute) { - if (!isset($this->eavAttributes[$attributeType][$attribute]['attribute_id'])) { - if (isset($this->eavAttributes[$attributeType])) { - $message = sprintf('Attribute %s does not exist in the type %s', $attribute, $attributeType); - } else { - $message = sprintf('Attribute type %s does not exist', $attributeType); - } - throw new \Migration\Exception($message); - } - $attributeId = $this->eavAttributes[$attributeType][$attribute]['attribute_id']; - $attributeCode = $this->eavAttributes[$attributeType][$attribute]['attribute_code']; - $this->skipAttributes[$attributeType][$attributeId] = $attributeCode; - } - } - - /** - * @param string $attributeType - * @return void + * Retrieves data for all attributes relative to $entityTypeCode entity type + * + * @param string $entityTypeCode + * @return array */ - protected function initEavAttributes($attributeType) + protected function getAttributesData($entityTypeCode) { - if (isset($this->eavAttributes[$attributeType])) { - return; - } + if (!isset($this->eavAttributes[$entityTypeCode])) { + $this->eavAttributes[$entityTypeCode] = []; + /** @var Mysql $adapter */ + $adapter = $this->source->getAdapter(); + $query = $adapter->getSelect() + ->from( + ['et' => $this->source->addDocumentPrefix('eav_entity_type')], + [] + )->join( + ['ea' => $this->source->addDocumentPrefix('eav_attribute')], + 'et.entity_type_id = ea.entity_type_id', + [ + 'attribute_id', + 'backend_type', + 'attribute_code', + 'entity_type_id' + ] + )->where( + 'et.entity_type_code = ?', + $entityTypeCode + ); + $attributes = $query->getAdapter()->fetchAll($query); - /** @var Mysql $adapter */ - $adapter = $this->source->getAdapter(); - $query = $adapter->getSelect() - ->from( - ['et' => $this->source->addDocumentPrefix('eav_entity_type')], - [] - )->join( - ['ea' => $this->source->addDocumentPrefix('eav_attribute')], - 'et.entity_type_id = ea.entity_type_id', - [ - 'attribute_id', - 'backend_type', - 'attribute_code', - 'entity_type_id' - ] - )->where( - 'et.entity_type_code = ?', - $attributeType - ); - $attributes = $query->getAdapter()->fetchAll($query); - - foreach ($attributes as $attribute) { - $this->eavAttributes[$attributeType][$attribute['attribute_code']] = $attribute; - $this->eavAttributes[$attributeType][$attribute['attribute_id']] = $attribute; + foreach ($attributes as $attribute) { + $this->eavAttributes[$entityTypeCode][$attribute['attribute_code']] = $attribute; + } } + return $this->eavAttributes[$entityTypeCode]; } /** diff --git a/src/Migration/Step/Customer/Integrity.php b/src/Migration/Step/Customer/Integrity.php index 394176ac5..c1c6dd997 100644 --- a/src/Migration/Step/Customer/Integrity.php +++ b/src/Migration/Step/Customer/Integrity.php @@ -63,18 +63,20 @@ public function __construct( */ public function perform() { - $messages = []; $this->progress->start($this->getIterationsCount()); - $srcDocuments = array_keys($this->readerGroups->getGroup('source_documents')); + + $attributesError = false; + try { + $this->helper->getSkippedAttributes(); + } catch (\Migration\Exception $e) { + $this->logger->error($e->getMessage()); + $attributesError = true; + } $dstDocuments = []; + $srcDocuments = array_keys($this->readerGroups->getGroup('source_documents')); foreach ($srcDocuments as $sourceDocumentName) { $dstDocuments[] = $this->map->getDocumentMap($sourceDocumentName, MapInterface::TYPE_SOURCE); - try { - $this->helper->getAttributeType($sourceDocumentName); - } catch (\Migration\Exception $e) { - $messages[] = $e->getMessage(); - } $this->progress->advance(); } @@ -82,11 +84,8 @@ public function perform() $this->check($dstDocuments, MapInterface::TYPE_DEST); $this->progress->finish(); - foreach ($messages as $message) { - $this->logger->error($message); - } - return $this->checkForErrors() && empty($messages); + return $this->checkForErrors() && $attributesError === false; } /** diff --git a/tests/unit/testsuite/Migration/Step/Customer/HelperTest.php b/tests/unit/testsuite/Migration/Step/Customer/HelperTest.php index 1dd858340..d527eba82 100644 --- a/tests/unit/testsuite/Migration/Step/Customer/HelperTest.php +++ b/tests/unit/testsuite/Migration/Step/Customer/HelperTest.php @@ -5,27 +5,15 @@ */ namespace unit\testsuite\Migration\Step\Customer; +use Migration\ResourceModel\Structure; +use Migration\ResourceModel\Record; +use Migration\ResourceModel\Record\Collection; use Migration\Step\Customer\Helper; class HelperTest extends \PHPUnit_Framework_TestCase { /** - * Entity name - */ - const ENTITY = 'customer'; - - /** - * Document name - */ - const DOCUMENT = 'customer_entity'; - - /** - * Attribute name - */ - const ATTRIBUTE = 'password_hash'; - - /** - * @var Helper + * @var Helper|\PHPUnit_Framework_MockObject_MockObject */ protected $helper; @@ -57,75 +45,77 @@ class HelperTest extends \PHPUnit_Framework_TestCase /** * @var \Migration\ResourceModel\Adapter\Mysql|\PHPUnit_Framework_MockObject_MockObject */ - protected $adapter; + protected $sourceAdapter; /** - * @var \Magento\Framework\DB\Select|\PHPUnit_Framework_MockObject_MockObject - */ - protected $select; - - /** - * @var array + * @var \Migration\ResourceModel\Adapter\Mysql|\PHPUnit_Framework_MockObject_MockObject */ - protected $sourceDocuments = [ - self::DOCUMENT => 'entity_id' - ]; + protected $destAdapter; /** - * @var array + * @var \Magento\Framework\DB\Select|\PHPUnit_Framework_MockObject_MockObject */ - protected $attribute = [ - [ - 'attribute_id' => '12', - 'backend_type' => 'varchar', - 'attribute_code' => self::ATTRIBUTE, - 'entity_type_id' => '1' - ] - ]; + protected $select; /** * @return void */ public function setUp() { - $this->adapter = $this->getMockBuilder('Migration\ResourceModel\Adapter\Mysql') + $this->sourceAdapter = $this->getMockBuilder(\Migration\ResourceModel\Adapter\Mysql::class) + ->disableOriginalConstructor() ->setMethods(['fetchAll', 'getSelect']) + ->getMock(); + $this->destAdapter = $this->getMockBuilder(\Migration\ResourceModel\Adapter\Mysql::class) ->disableOriginalConstructor() + ->setMethods(['getDocumentStructure']) ->getMock(); + $this->select = $this->getMockBuilder('Magento\Framework\DB\Select') - ->setMethods(['from', 'join', 'where', 'getAdapter']) + ->setMethods(['from', 'join', 'where', 'union', 'getAdapter']) ->disableOriginalConstructor() ->getMock(); $this->select->expects($this->any())->method('from')->willReturnSelf(); $this->select->expects($this->any())->method('join')->willReturnSelf(); $this->select->expects($this->any())->method('where')->willReturnSelf(); - $this->select->expects($this->any())->method('getAdapter')->willReturn($this->adapter); - $this->adapter->expects($this->any())->method('getSelect')->willReturn($this->select); + $this->select->expects($this->any())->method('union')->willReturnSelf(); + $this->select->expects($this->any())->method('getAdapter')->willReturn($this->sourceAdapter); + $this->sourceAdapter->expects($this->any())->method('getSelect')->willReturn($this->select); - $this->source = $this->getMockBuilder('Migration\ResourceModel\Source') + $this->source = $this->getMockBuilder(\Migration\ResourceModel\Source::class) ->disableOriginalConstructor() + ->setMethods(['getAdapter', 'addDocumentPrefix']) ->getMock(); - $this->destination = $this->getMockBuilder('Migration\ResourceModel\Destination') + $this->source->expects($this->any())->method('getAdapter')->willReturn($this->sourceAdapter); + $this->source->expects($this->any())->method('addDocumentPrefix')->willReturnArgument(0); + + $this->destination = $this->getMockBuilder(\Migration\ResourceModel\Destination::class) ->disableOriginalConstructor() + ->setMethods(['getAdapter']) ->getMock(); - $this->configReader = $this->getMockBuilder('Migration\Config') + $this->destination->expects($this->any())->method('getAdapter')->willReturn($this->destAdapter); + + $this->configReader = $this->getMockBuilder(\Migration\Config::class) ->disableOriginalConstructor() ->getMock(); - $this->readerAttributes = $this->getMockBuilder('Migration\Reader\Groups') + $this->readerAttributes = $this->getMockBuilder(\Migration\Reader\Groups::class) ->disableOriginalConstructor() ->getMock(); - $this->readerGroups = $this->getMockBuilder('Migration\Reader\Groups') + $this->readerGroups = $this->getMockBuilder(\Migration\Reader\Groups::class) ->setMethods(['getGroup']) ->disableOriginalConstructor() ->getMock(); $this->readerGroups->expects($this->at(0)) ->method('getGroup') ->with('source_documents') - ->willReturn($this->sourceDocuments); + ->willReturn([ + 'customer_entity' => 'entity_id', + 'customer_address_entity' => 'entity_id' + ]); - $groupsFactory = $this->getMockBuilder('Migration\Reader\GroupsFactory') + $groupsFactory = $this->getMockBuilder(\Migration\Reader\GroupsFactory::class) ->setMethods(['create']) ->disableOriginalConstructor() ->getMock(); @@ -138,160 +128,250 @@ public function setUp() ->with('customer_document_groups_file') ->willReturn($this->readerGroups); - $this->helper = new Helper( - $this->source, - $this->destination, - $groupsFactory, - $this->configReader - ); + $this->helper = $this->getMockBuilder(Helper::class) + ->setConstructorArgs([$this->source, $this->destination, $groupsFactory, $this->configReader]) + ->setMethods(['getAttributesData']) + ->getMock(); } /** - * @param array $attributeData - * @param array $expected + * @param array $testMethodArguments + * @param array $recordsData + * @param array $attributesValues + * @param array $upgradePasswordHash + * @param array $recordsResult * - * @dataProvider dataProviderUpdateAttributeData + * @dataProvider fixturesDataProvider * @return void */ - public function testUpdateAttributeData($attributeData, $expected) - { - $this->getAttributeType(); - - $this->readerAttributes->expects($this->any()) - ->method('getGroup') - ->with(self::DOCUMENT) - ->willReturn($this->attribute); - - $this->adapter->expects($this->at(2))->method('fetchAll')->with($this->select)->willReturn($attributeData); - - $structure = $this->getMockBuilder('Migration\ResourceModel\Structure') - ->disableOriginalConstructor() - ->getMock(); - $record = $this->getMockBuilder('Migration\ResourceModel\Record') + public function testUpdateAttributeData( + $testMethodArguments, + $recordsData, + $attributesValues, + $upgradePasswordHash, + $recordsResult + ) { + $records = []; + foreach ($recordsData as $recordData) { + $records[] = $this->getMockBuilder(Record::class) + ->setConstructorArgs([$recordData]) + ->setMethods(null) + ->getMock(); + } + $structure = $this->getMockBuilder(Structure::class) ->disableOriginalConstructor() ->getMock(); + $destinationRecords = new Collection($structure, $records); + + $this->initMaps(); - $this->configReader->expects($this->any())->method('getOption') - ->willReturn(current($attributeData)[Helper::UPGRADE_CUSTOMER_PASSWORD_HASH]); + $this->sourceAdapter + ->expects($this->once()) + ->method('fetchAll') + ->with($this->select) + ->willReturn($attributesValues); - $record->expects($this->any())->method('getValue')->with('entity_id')->willReturn('1'); - $record->expects($this->any())->method('getData')->willReturn([]); - $record->expects($this->any())->method('setData')->with($expected); - $destinationRecords = new \Migration\ResourceModel\Record\Collection($structure, [$record]); + $this->configReader + ->expects($this->any()) + ->method('getOption') + ->willReturnMap([$upgradePasswordHash]); - $this->helper->updateAttributeData(self::ENTITY, self::DOCUMENT, $destinationRecords); + $testMethodArguments[] = $destinationRecords; + $this->helper->updateAttributeData(...$testMethodArguments); + + foreach ($destinationRecords as $key => $record) { + $this->assertEquals($record->getData(), $recordsResult[$key]); + } } /** - * Init EAV attributes + * Initializes Mocks return maps + * * @return void */ - protected function getAttributeType() + private function initMaps() { - $entities = [self::ENTITY => '']; - $documentGroups = [self::ATTRIBUTE => '']; - - $this->readerGroups->expects($this->at(0)) - ->method('getGroup') - ->with('eav_entities') - ->willReturn($entities); - $this->readerGroups->expects($this->at(1)) + $this->helper + ->expects($this->any()) + ->method('getAttributesData') + ->willReturnMap([ + [ + 'customer', + [ + 'password_hash' => [ + 'attribute_id' => '12', + 'backend_type' => 'varchar', + 'attribute_code' => 'password_hash', + 'entity_type_id' => '1' + ], + ] + ], [ + 'customer_address', + [ + 'company' => [ + 'attribute_id' => '13', + 'backend_type' => 'varchar', + 'attribute_code' => 'company', + 'entity_type_id' => '2' + ], + 'country_id' => [ + 'attribute_id' => '14', + 'backend_type' => 'varchar', + 'attribute_code' => 'country_id', + 'entity_type_id' => '2' + ], + ] + ] + ]); + $this->readerAttributes + ->expects($this->any()) ->method('getGroup') - ->with(self::ENTITY) - ->willReturn($this->sourceDocuments); - - $this->source->expects($this->any())->method('getAdapter')->willReturn($this->adapter); - $this->adapter->expects($this->at(1))->method('fetchAll')->with($this->select)->willReturn($this->attribute); - - $this->readerAttributes->expects($this->any())->method('getGroup')->willReturn($documentGroups); - - $this->helper->getAttributeType(self::DOCUMENT); + ->willReturnMap([ + ['customer_entity', ['entity_id' => '', 'entity_type_id' => '', 'email' => '', 'password_hash' => '']], + ['customer_address_entity', ['entity_id' => '', 'city' => '', 'company' => '', 'country_id' => '']] + ]); + $this->destAdapter + ->expects($this->any()) + ->method('getDocumentStructure') + ->willReturnMap([ + [ + 'customer_entity', + [ + 'entity_id' => ['DEFAULT' => null, 'NULLABLE' => false], + 'entity_type_id' => ['DEFAULT' => null, 'NULLABLE' => false], + 'email' => ['DEFAULT' => null, 'NULLABLE' => true], + 'password_hash' => ['DEFAULT' => null, 'NULLABLE' => false], + ] + ], [ + 'customer_address_entity', + [ + 'entity_id' => ['DEFAULT' => null, 'NULLABLE' => false], + 'city' => ['DEFAULT' => null, 'NULLABLE' => false], + 'company' => ['DEFAULT' => null, 'NULLABLE' => true], + 'country_id' => ['DEFAULT' => null, 'NULLABLE' => false], + ] + ] + ]); } /** * @return array */ - public function dataProviderUpdateAttributeData() + public function fixturesDataProvider() { return [ [ - [ + 'testMethodArguments' => ['customer', 'customer_entity', 'customer_entity'], + 'recordsData' => [ + ['entity_id' => '1', 'entity_type_id' => '1', 'email' => 'customer1@example.com'], + ['entity_id' => '2', 'entity_type_id' => '1', 'email' => 'customer2@example.com'], + ['entity_id' => '3', 'entity_type_id' => '1', 'email' => 'customer3@example.com'], + ['entity_id' => '4', 'entity_type_id' => '1', 'email' => 'customer4@example.com'] + ], + 'attributesValues' => [ [ - Helper::UPGRADE_CUSTOMER_PASSWORD_HASH => 1, 'entity_id' => '1', 'attribute_id' => '12', 'value' => '34356a3d028accfb3c2996827b706bf5:UmPvGtih25eQCjC5f6NMwqkds500x2Jd' + ], [ + 'entity_id' => '2', + 'attribute_id' => '12', + 'value' => '86a375aacb17606c185d31c8d3e320ce' + ], [ + 'entity_id' => '3', + 'attribute_id' => '12', + 'value' => 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' + ], [ + 'entity_id' => '4', + 'attribute_id' => '12', + 'value' => '123123q:UmPvGtih25eQCjC5f6NMwqkds500x2Jd' ] ], - [ - self::ATTRIBUTE => '34356a3d028accfb3c2996827b706bf5:UmPvGtih25eQCjC5f6NMwqkds500x2Jd:0' - ] - ], - [ - [ + 'upgradePasswordHash' => [Helper::UPGRADE_CUSTOMER_PASSWORD_HASH, true], + 'recordsResult' => [ [ - Helper::UPGRADE_CUSTOMER_PASSWORD_HASH => 1, 'entity_id' => '1', - 'attribute_id' => '12', - 'value' => '86a375aacb17606c185d31c8d3e320ce' + 'entity_type_id' => '1', + 'email' => 'customer1@example.com', + 'password_hash' => '34356a3d028accfb3c2996827b706bf5:UmPvGtih25eQCjC5f6NMwqkds500x2Jd:0' + ], [ + 'entity_id' => '2', + 'entity_type_id' => '1', + 'email' => 'customer2@example.com', + 'password_hash' => '86a375aacb17606c185d31c8d3e320ce::0' + ], [ + 'entity_id' => '3', + 'entity_type_id' => '1', + 'email' => 'customer3@example.com', + 'password_hash' => 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855::1' + ], [ + 'entity_id' => '4', + 'entity_type_id' => '1', + 'email' => 'customer4@example.com', + 'password_hash' => '123123q:UmPvGtih25eQCjC5f6NMwqkds500x2Jd' ] ], - [ - self::ATTRIBUTE => '86a375aacb17606c185d31c8d3e320ce::0' - ] - ], - [ - [ + ], [ + 'testMethodArguments' => ['customer', 'customer_entity', 'customer_entity'], + 'recordsData' => [ + ['entity_id' => '1', 'entity_type_id' => '1', 'email' => 'customer1@example.com'], + ['entity_id' => '2', 'entity_type_id' => '1', 'email' => 'customer2@example.com'] + ], + 'attributesValues' => [ [ - Helper::UPGRADE_CUSTOMER_PASSWORD_HASH => 1, 'entity_id' => '1', 'attribute_id' => '12', - 'value' => 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' + 'value' => '34356a3d028accfb3c2996827b706bf5:UmPvGtih25eQCjC5f6NMwqkds500x2Jd' + ], + [ + 'entity_id' => '2', + 'attribute_id' => '12', + 'value' => '123123q:UmPvGtih25eQCjC5f6NMwqkds500x2Jd' ] ], - [ - self::ATTRIBUTE => 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855::1' - ] - ], - [ - [ + 'upgradePasswordHash' => [Helper::UPGRADE_CUSTOMER_PASSWORD_HASH, false], + 'recordsResult' => [ [ - Helper::UPGRADE_CUSTOMER_PASSWORD_HASH => 1, 'entity_id' => '1', - 'attribute_id' => '12', - 'value' => '123123q:UmPvGtih25eQCjC5f6NMwqkds500x2Jd' + 'entity_type_id' => '1', + 'email' => 'customer1@example.com', + 'password_hash' => '34356a3d028accfb3c2996827b706bf5:UmPvGtih25eQCjC5f6NMwqkds500x2Jd' + ], + [ + 'entity_id' => '2', + 'entity_type_id' => '1', + 'email' => 'customer2@example.com', + 'password_hash' => '123123q:UmPvGtih25eQCjC5f6NMwqkds500x2Jd' ] ], - [ - self::ATTRIBUTE => '123123q:UmPvGtih25eQCjC5f6NMwqkds500x2Jd' - ] - ], - [ - [ + ], [ + 'testMethodArguments' => ['customer_address', 'customer_address_entity', 'customer_address_entity'], + 'recordsData' => [ + ['entity_id' => '1', 'city' => 'Austin'], + ['entity_id' => '2', 'city' => 'Kiev'] + ], + 'attributesValues' => [ [ - Helper::UPGRADE_CUSTOMER_PASSWORD_HASH => 0, 'entity_id' => '1', - 'attribute_id' => '12', - 'value' => '34356a3d028accfb3c2996827b706bf5:UmPvGtih25eQCjC5f6NMwqkds500x2Jd' + 'attribute_id' => '13', + 'value' => 'Company Name' ] ], - [ - self::ATTRIBUTE => '34356a3d028accfb3c2996827b706bf5:UmPvGtih25eQCjC5f6NMwqkds500x2Jd' - ] - ], - [ - [ + 'upgradePasswordHash' => [Helper::UPGRADE_CUSTOMER_PASSWORD_HASH, false], + 'recordsResult' => [ [ - Helper::UPGRADE_CUSTOMER_PASSWORD_HASH => 0, 'entity_id' => '1', - 'attribute_id' => '12', - 'value' => '123123q:UmPvGtih25eQCjC5f6NMwqkds500x2Jd' + 'city' => 'Austin', + 'company' => 'Company Name', + 'country_id' => '' + ], [ + 'entity_id' => '2', + 'city' => 'Kiev', + 'company' => null, + 'country_id' => '' ] ], - [ - self::ATTRIBUTE => '123123q:UmPvGtih25eQCjC5f6NMwqkds500x2Jd' - ] - ], + ] ]; } } From a771dabc89c9cef54101c779a80f90c829a0ce9a Mon Sep 17 00:00:00 2001 From: Victor Rad Date: Tue, 24 Jan 2017 10:56:53 +0200 Subject: [PATCH 02/17] MAGETWO-63552: SQL initStatements should not be a part of PDO DSN --- .../ResourceModel/Adapter/Pdo/MysqlBuilder.php | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/Migration/ResourceModel/Adapter/Pdo/MysqlBuilder.php b/src/Migration/ResourceModel/Adapter/Pdo/MysqlBuilder.php index ab26e25b1..aef5d0319 100644 --- a/src/Migration/ResourceModel/Adapter/Pdo/MysqlBuilder.php +++ b/src/Migration/ResourceModel/Adapter/Pdo/MysqlBuilder.php @@ -65,6 +65,7 @@ public function build($resourceType) ] ); $instance->disallowDdlCache(); + $this->runInitStatements($instance, $resourceType); return $instance; } @@ -83,11 +84,23 @@ private function getConfig($resourceType) $config['username'] = $resource[$type]['user']; $config['password'] = !empty($resource[$type]['password']) ? $resource[$type]['password'] : ''; + return $config; + } + + /** + * Run init SQL statements + * + * @param \Magento\Framework\DB\Adapter\Pdo\Mysql $instance + * @param string $resourceType + * @return void + */ + private function runInitStatements(Mysql $instance, $resourceType) + { $initStatements = $this->config->getOption('init_statements_' . $resourceType); if (!empty($initStatements)) { - $config['initStatements'] = $initStatements; + $instance->query($initStatements); } - return $config; + return; } /** From 360132a48d5478799e5fb1c07c9bb9db863ce1d2 Mon Sep 17 00:00:00 2001 From: Victor Rad Date: Tue, 24 Jan 2017 11:00:41 +0200 Subject: [PATCH 03/17] MAGETWO-63223: [GitHub] Can't specify port different than default #221 --- etc/config.xsd | 1 + src/Migration/ResourceModel/Adapter/Pdo/MysqlBuilder.php | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/etc/config.xsd b/etc/config.xsd index 4dcd11bd6..ee0125b03 100644 --- a/etc/config.xsd +++ b/etc/config.xsd @@ -72,6 +72,7 @@ + diff --git a/src/Migration/ResourceModel/Adapter/Pdo/MysqlBuilder.php b/src/Migration/ResourceModel/Adapter/Pdo/MysqlBuilder.php index aef5d0319..09ac542ba 100644 --- a/src/Migration/ResourceModel/Adapter/Pdo/MysqlBuilder.php +++ b/src/Migration/ResourceModel/Adapter/Pdo/MysqlBuilder.php @@ -83,7 +83,9 @@ private function getConfig($resourceType) $config['dbname'] = $resource[$type]['name']; $config['username'] = $resource[$type]['user']; $config['password'] = !empty($resource[$type]['password']) ? $resource[$type]['password'] : ''; - + if (!empty($resource[$type]['port'])) { + $config['port'] = $resource[$type]['port']; + } return $config; } From 3e9fe538873599edc5ad1a16680015f72943ff0b Mon Sep 17 00:00:00 2001 From: Victor Rad Date: Tue, 24 Jan 2017 13:19:22 +0200 Subject: [PATCH 04/17] MAGETWO-63223: [GitHub] Can't specify port different than default #221 --- .../Adapter/Pdo/MysqlBuilder.php | 10 +- .../Adapter/Pdo/MysqlBuilderTest.php | 110 ++++++++++++++++++ 2 files changed, 115 insertions(+), 5 deletions(-) create mode 100644 tests/unit/testsuite/Migration/ResourceModel/Adapter/Pdo/MysqlBuilderTest.php diff --git a/src/Migration/ResourceModel/Adapter/Pdo/MysqlBuilder.php b/src/Migration/ResourceModel/Adapter/Pdo/MysqlBuilder.php index aef5d0319..43137a9b2 100644 --- a/src/Migration/ResourceModel/Adapter/Pdo/MysqlBuilder.php +++ b/src/Migration/ResourceModel/Adapter/Pdo/MysqlBuilder.php @@ -6,12 +6,12 @@ namespace Migration\ResourceModel\Adapter\Pdo; use Magento\Framework\ObjectManagerInterface; -use Magento\Framework\DB\Adapter\Pdo\Mysql; +use Magento\Framework\DB\Adapter\Pdo\Mysql as PdoMysql; use Magento\Framework\DB\SelectFactory; use Migration\Config; /** - * Builder class for @see Mysql + * Builder class for @see PdoMysql */ class MysqlBuilder { @@ -53,7 +53,7 @@ public function __construct( * Create class instance with specified parameters * * @param string $resourceType - * @return Mysql + * @return PdoMysql */ public function build($resourceType) { @@ -70,7 +70,7 @@ public function build($resourceType) } /** - * Returns well-formed configuration array of $resourceType resource for @see Mysql + * Returns well-formed configuration array of $resourceType resource for @see PdoMysql * * @param string $resourceType * @return array @@ -94,7 +94,7 @@ private function getConfig($resourceType) * @param string $resourceType * @return void */ - private function runInitStatements(Mysql $instance, $resourceType) + private function runInitStatements(PdoMysql $instance, $resourceType) { $initStatements = $this->config->getOption('init_statements_' . $resourceType); if (!empty($initStatements)) { diff --git a/tests/unit/testsuite/Migration/ResourceModel/Adapter/Pdo/MysqlBuilderTest.php b/tests/unit/testsuite/Migration/ResourceModel/Adapter/Pdo/MysqlBuilderTest.php new file mode 100644 index 000000000..a967f65e6 --- /dev/null +++ b/tests/unit/testsuite/Migration/ResourceModel/Adapter/Pdo/MysqlBuilderTest.php @@ -0,0 +1,110 @@ +objectManager = $this->getMockBuilder('\Magento\Framework\ObjectManager\ObjectManager') + ->disableOriginalConstructor() + ->setMethods(['create']) + ->getMock(); + $this->config = $this->getMockBuilder('Migration\Config') + ->disableOriginalConstructor() + ->setMethods(['getResourceConfig', 'getOption']) + ->getMock(); + $this->selectFactory = $this->getMockBuilder('Magento\Framework\DB\SelectFactory') + ->disableOriginalConstructor() + ->getMock(); + $this->pdoMysql = $this->getMockBuilder('\Magento\Framework\DB\Adapter\Pdo\Mysql') + ->disableOriginalConstructor() + ->setMethods(['disallowDdlCache', 'query']) + ->getMock(); + $this->mysqlBuilder = new MysqlBuilder($this->objectManager, $this->config); + } + + /** + * @return void + */ + public function testBuild() + { + $resourceType = 'source'; + $initStatements = 'SET NAMES utf8;'; + $resourceConfig = [ + 'type' => $resourceType, + $resourceType => [ + 'host' => 'localhost', + 'name' => 'db1', + 'user' => 'root', + 'password' => 'root' + ], + ]; + $mysqlPdoConfig = ['host' => 'localhost', 'dbname' => 'db1', 'username' => 'root', 'password' => 'root']; + $this->config->expects($this->once()) + ->method('getResourceConfig') + ->with($resourceType) + ->willReturn($resourceConfig); + $this->config->expects($this->any())->method('getOption')->willReturnMap([ + ['edition_migrate', 'ce-to-ce'], + ['init_statements_' . $resourceType, $initStatements] + ]); + $this->objectManager->expects($this->at(0)) + ->method('create') + ->with('\Magento\Framework\DB\SelectFactory', ['parts' => []]) + ->willReturn($this->selectFactory); + $this->objectManager->expects($this->at(1)) + ->method('create') + ->with( + '\Magento\Framework\DB\Adapter\Pdo\Mysql', + ['config' => $mysqlPdoConfig, 'selectFactory' => $this->selectFactory] + ) + ->willReturn($this->pdoMysql); + $this->pdoMysql->expects($this->once()) + ->method('disallowDdlCache') + ->willReturnSelf(); + $this->pdoMysql->expects($this->once()) + ->method('query') + ->with($initStatements) + ->willReturnSelf(); + + $this->assertEquals($this->pdoMysql, $this->mysqlBuilder->build($resourceType)); + } +} From cd532e61019f240372f4945910e6f5da0ba397f7 Mon Sep 17 00:00:00 2001 From: Victor Rad Date: Wed, 25 Jan 2017 15:27:00 +0200 Subject: [PATCH 05/17] MAGETWO-63223: [GitHub] Can't specify port different than default #221 --- .../ResourceModel/Adapter/Pdo/MysqlBuilderTest.php | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/unit/testsuite/Migration/ResourceModel/Adapter/Pdo/MysqlBuilderTest.php b/tests/unit/testsuite/Migration/ResourceModel/Adapter/Pdo/MysqlBuilderTest.php index a967f65e6..e097b3921 100644 --- a/tests/unit/testsuite/Migration/ResourceModel/Adapter/Pdo/MysqlBuilderTest.php +++ b/tests/unit/testsuite/Migration/ResourceModel/Adapter/Pdo/MysqlBuilderTest.php @@ -72,12 +72,19 @@ public function testBuild() 'type' => $resourceType, $resourceType => [ 'host' => 'localhost', + 'port' => '9999', 'name' => 'db1', 'user' => 'root', 'password' => 'root' ], ]; - $mysqlPdoConfig = ['host' => 'localhost', 'dbname' => 'db1', 'username' => 'root', 'password' => 'root']; + $mysqlPdoConfig = [ + 'host' => 'localhost', + 'port' => '9999', + 'dbname' => 'db1', + 'username' => 'root', + 'password' => 'root' + ]; $this->config->expects($this->once()) ->method('getResourceConfig') ->with($resourceType) From 2ddd41b33c75917501b7c0e5970766cd34131979 Mon Sep 17 00:00:00 2001 From: Victor Rad Date: Fri, 3 Feb 2017 13:57:36 +0200 Subject: [PATCH 06/17] MAGETWO-63104: [GitHub] System attributes are not linked with attribute sets #212 --- src/Migration/Step/Eav/Data.php | 59 +++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 22 deletions(-) diff --git a/src/Migration/Step/Eav/Data.php b/src/Migration/Step/Eav/Data.php index aa1ff6fbb..a56d6ffb9 100644 --- a/src/Migration/Step/Eav/Data.php +++ b/src/Migration/Step/Eav/Data.php @@ -122,6 +122,20 @@ class Data implements StageInterface, RollbackInterface ] ]; + /** + * Attributes will be added to attribute group if not exist + * + * @var array + */ + private $attributesGroupToAdd = [ + 'category_ids' => 'product-details', + 'price_type' => 'product-details', + 'sku_type' => 'product-details', + 'weight_type' => 'product-details', + 'giftcard_type' => 'product-details', + 'quantity_and_stock_status' => 'product-details' + ]; + /** * @param Source $source * @param Destination $destination @@ -493,7 +507,9 @@ private function moveProductAttributes($recordsToSave) { $this->moveProductAttributeToGroup($recordsToSave, 'price', 'product-details'); $this->moveProductAttributeToGroup($recordsToSave, 'shipment_type', 'bundle-items'); - $this->addProductAttributeToGroup($recordsToSave, 'quantity_and_stock_status', 'product-details'); + foreach ($this->attributesGroupToAdd as $attributeCode => $attributeGroupCode) { + $this->addProductAttributeToGroup($recordsToSave, $attributeCode, $attributeGroupCode); + } return $recordsToSave; } @@ -548,37 +564,37 @@ private function addProductAttributeToGroup($recordsToSave, $attributeCode, $att { $productEntityType = $this->helper->getSourceRecords('eav_entity_type', ['entity_type_code'])['catalog_product']; + $productEntityTypeId = $productEntityType['entity_type_id']; $attributes = $this->helper->getDestinationRecords('eav_attribute', ['attribute_id']); $attributeGroups = $this->helper->getDestinationRecords('eav_attribute_group', ['attribute_group_id']); - $attributeSetGroups = []; - foreach ($attributeGroups as $attributeGroup) { - if ($attributeGroup['attribute_group_code'] == $attributeGroupCode) { - $attributeSetGroups[$attributeGroup['attribute_set_id']][$attributeGroupCode] = - $attributeGroup['attribute_group_id']; - } - } + $attributeSets = $this->helper->getDestinationRecords('eav_attribute_set', ['attribute_set_id']); + $attributeSetGroupsFound = []; $attribute = null; + $destinationDocument = $this->destination->getDocument( + $this->map->getDocumentMap('eav_entity_attribute', MapInterface::TYPE_SOURCE) + ); foreach ($recordsToSave as $record) { $attributeId = $record->getValue('attribute_id'); $entityTypeId = $record->getValue('entity_type_id'); - if (!isset($attributes[$attributeId]) - || $entityTypeId != $productEntityType['entity_type_id'] + if (isset($attributes[$attributeId]) + && $attributes[$attributeId]['attribute_code'] == $attributeCode + && $entityTypeId == $productEntityTypeId ) { - continue; - } - if ($attributes[$attributeId]['attribute_code'] == $attributeCode) { - $attributeSetGroups[$record->getValue('attribute_set_id')][$attributeCode] = + $attributeSetGroupsFound[$record->getValue('attribute_set_id')] = $record->getValue('attribute_group_id'); $attribute = $record->getData(); } } - $destinationDocument = $this->destination->getDocument( - $this->map->getDocumentMap('eav_entity_attribute', MapInterface::TYPE_SOURCE) - ); - foreach ($attributeSetGroups as $attributeSetId => $attributeSetGroup) { - if (!isset($attributeSetGroup[$attributeCode])) { - $attribute['attribute_set_id'] = $attributeSetId; - $attribute['attribute_group_id'] = $attributeSetGroup[$attributeGroupCode]; + if ($attribute === null) { + return $recordsToSave; + } + foreach ($attributeGroups as $attributeGroup) { + if ($attributeGroup['attribute_group_code'] == $attributeGroupCode + && $attributeSets[$attributeGroup['attribute_set_id']]['entity_type_id'] == $productEntityTypeId + && !isset($attributeSetGroupsFound[$attributeGroup['attribute_set_id']]) + ) { + $attribute['attribute_set_id'] = $attributeGroup['attribute_set_id']; + $attribute['attribute_group_id'] = $attributeGroup['attribute_group_id']; $attribute['entity_attribute_id'] = null; $destinationRecord = $this->factory->create( [ @@ -589,7 +605,6 @@ private function addProductAttributeToGroup($recordsToSave, $attributeCode, $att $recordsToSave->addRecord($destinationRecord); } } - return $recordsToSave; } From cf7512b671375d7f12b93e7544ed5189601d178e Mon Sep 17 00:00:00 2001 From: Victor Rad Date: Wed, 15 Feb 2017 11:00:50 +0200 Subject: [PATCH 07/17] MAGETWO-62034: [DB-PPD] Data migration fails on Customer Step --- src/Migration/Step/Customer/Helper.php | 33 ++++++++++++-------------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/src/Migration/Step/Customer/Helper.php b/src/Migration/Step/Customer/Helper.php index 9cd3d094b..bededba93 100644 --- a/src/Migration/Step/Customer/Helper.php +++ b/src/Migration/Step/Customer/Helper.php @@ -160,25 +160,22 @@ public function getSkippedAttributes() foreach ($entityTypeCodes as $entityTypeCode) { $documents = $this->readerGroups->getGroup($entityTypeCode); foreach ($documents as $documentName => $key) { - if ($key == 'entity_id') { - foreach (array_keys($this->readerAttributes->getGroup($documentName)) as $attributeCode) { - $eavAttributes = $this->getAttributesData($entityTypeCode); - if (empty($eavAttributes)) { - throw new \Migration\Exception( - sprintf('Attribute type %s does not exist', $entityTypeCode) - ); - } elseif (!isset($eavAttributes[$attributeCode])) { - throw new \Migration\Exception( - sprintf( - 'Attribute %s does not exist in the type %s', - $attributeCode, - $entityTypeCode - ) - ); - } - $attributeId = $eavAttributes[$attributeCode]['attribute_id']; - $this->skipAttributes[$entityTypeCode][$attributeId] = $attributeCode; + if ($key != 'entity_id') { + continue; + } + foreach (array_keys($this->readerAttributes->getGroup($documentName)) as $attributeCode) { + $eavAttributes = $this->getAttributesData($entityTypeCode); + if (!isset($eavAttributes[$attributeCode])) { + throw new \Migration\Exception( + sprintf( + 'Attribute %s does not exist in the type %s', + $attributeCode, + $entityTypeCode + ) + ); } + $attributeId = $eavAttributes[$attributeCode]['attribute_id']; + $this->skipAttributes[$entityTypeCode][$attributeId] = $attributeCode; } } } From 1ef51153c008a9fa6c1770878f3643f6b31ed7e0 Mon Sep 17 00:00:00 2001 From: Victor Rad Date: Fri, 17 Feb 2017 10:57:30 +0200 Subject: [PATCH 08/17] MAGETWO-63223: [GitHub] Can't specify port different than default #221 --- src/Migration/ResourceModel/Adapter/Pdo/MysqlBuilder.php | 1 - .../Migration/ResourceModel/Adapter/Pdo/MysqlBuilderTest.php | 5 ----- 2 files changed, 6 deletions(-) diff --git a/src/Migration/ResourceModel/Adapter/Pdo/MysqlBuilder.php b/src/Migration/ResourceModel/Adapter/Pdo/MysqlBuilder.php index a7f6e75b8..fdede6e7d 100644 --- a/src/Migration/ResourceModel/Adapter/Pdo/MysqlBuilder.php +++ b/src/Migration/ResourceModel/Adapter/Pdo/MysqlBuilder.php @@ -102,7 +102,6 @@ private function runInitStatements(PdoMysql $instance, $resourceType) if (!empty($initStatements)) { $instance->query($initStatements); } - return; } /** diff --git a/tests/unit/testsuite/Migration/ResourceModel/Adapter/Pdo/MysqlBuilderTest.php b/tests/unit/testsuite/Migration/ResourceModel/Adapter/Pdo/MysqlBuilderTest.php index e097b3921..6c7331a5d 100644 --- a/tests/unit/testsuite/Migration/ResourceModel/Adapter/Pdo/MysqlBuilderTest.php +++ b/tests/unit/testsuite/Migration/ResourceModel/Adapter/Pdo/MysqlBuilderTest.php @@ -13,11 +13,6 @@ class MysqlBuilderTest extends \PHPUnit_Framework_TestCase */ protected $pdoMysql; - /** - * @var \Migration\ResourceModel\Adapter\Mysql - */ - protected $adapterMysql; - /** * @var \Magento\Framework\ObjectManagerInterface|\PHPUnit_Framework_MockObject_MockObject */ From e3b92a54f65be28b8c5062694786da1ee25078e9 Mon Sep 17 00:00:00 2001 From: Victor Rad Date: Fri, 17 Feb 2017 14:21:33 +0200 Subject: [PATCH 09/17] MAGETWO-64150: [GitHub] swatch_image attribute is not a part of Migrated_Image attribute group #212 --- src/Migration/Model/Eav/AttributeGroupNameToCodeMap.php | 1 + src/Migration/Step/Eav/Data.php | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Migration/Model/Eav/AttributeGroupNameToCodeMap.php b/src/Migration/Model/Eav/AttributeGroupNameToCodeMap.php index c8e1075c7..83971e588 100644 --- a/src/Migration/Model/Eav/AttributeGroupNameToCodeMap.php +++ b/src/Migration/Model/Eav/AttributeGroupNameToCodeMap.php @@ -17,6 +17,7 @@ class AttributeGroupNameToCodeMap 'General' => 'product-details', 'Prices' => 'advanced-pricing', 'Design' => 'design', + 'Images' => 'image-management' ]; /** diff --git a/src/Migration/Step/Eav/Data.php b/src/Migration/Step/Eav/Data.php index a56d6ffb9..e1a508364 100644 --- a/src/Migration/Step/Eav/Data.php +++ b/src/Migration/Step/Eav/Data.php @@ -133,7 +133,8 @@ class Data implements StageInterface, RollbackInterface 'sku_type' => 'product-details', 'weight_type' => 'product-details', 'giftcard_type' => 'product-details', - 'quantity_and_stock_status' => 'product-details' + 'quantity_and_stock_status' => 'product-details', + 'swatch_image' => 'image-management' ]; /** From 74a1f36f2271e0c7c62fb33aefafb9e76ce400df Mon Sep 17 00:00:00 2001 From: Victor Rad Date: Mon, 27 Mar 2017 09:20:48 +0300 Subject: [PATCH 10/17] MAGETWO-56157: Unable to save Customer's custom Attributes --- src/Migration/Step/Eav/Data.php | 78 +++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/src/Migration/Step/Eav/Data.php b/src/Migration/Step/Eav/Data.php index bdb73eb85..a72ef93d7 100644 --- a/src/Migration/Step/Eav/Data.php +++ b/src/Migration/Step/Eav/Data.php @@ -493,6 +493,7 @@ protected function migrateEntityAttributes() $recordsToSave = $this->processDesignEntityAttributes($recordsToSave); $recordsToSave = $this->moveProductAttributes($recordsToSave); + $recordsToSave = $this->addCustomerAttributes($recordsToSave); $this->destination->clearDocument($destinationDocument->getName()); $this->saveRecords($destinationDocument, $recordsToSave); @@ -673,6 +674,83 @@ private function processDesignEntityAttributes($recordsToSave) return $recordsToSave; } + /** + * + * + * @param Record\Collection $recordsToSave + * @return Record\Collection + */ + private function addCustomerAttributes($recordsToSave) + { + $entityAttributeDocument = $this->destination->getDocument( + $this->map->getDocumentMap('eav_entity_attribute', MapInterface::TYPE_SOURCE) + ); + $customerEntityType + = $this->helper->getSourceRecords('eav_entity_type', ['entity_type_code'])['customer']; + $customerEntityTypeId = $customerEntityType['entity_type_id']; + $attributeGroups = $this->helper->getDestinationRecords('eav_attribute_group', ['attribute_group_id']); + $attributeSets = $this->helper->getDestinationRecords('eav_attribute_set', ['attribute_set_id']); + $attributeSetNameCustomerSource = 'Migration_Default'; + $attributeSetNameCustomerDestination = 'Default'; + $attributeSetIdOfCustomerSource = null; + $attributeSetIdOfCustomerDestination = null; + $eavEntityAttributeOfCustomerSource = []; + $eavEntityAttributeOfCustomerDestination = []; + $attributeGroupIfOfCustomerSource = null; + $recordToAdd = null; + + foreach ($attributeSets as $attributeSet) { + if ($attributeSet['entity_type_id'] == $customerEntityTypeId + && $attributeSet['attribute_set_name'] == $attributeSetNameCustomerSource + ) { + $attributeSetIdOfCustomerSource = $attributeSet['attribute_set_id']; + } elseif ($attributeSet['entity_type_id'] == $customerEntityTypeId + && $attributeSet['attribute_set_name'] == $attributeSetNameCustomerDestination + ) { + $attributeSetIdOfCustomerDestination = $attributeSet['attribute_set_id']; + } + } + foreach ($attributeGroups as $attributeGroup) { + if ($attributeGroup['attribute_set_id'] == $attributeSetIdOfCustomerSource) { + $attributeGroupIfOfCustomerSource = $attributeGroup['attribute_group_id']; + } + } + if ($attributeSetIdOfCustomerSource === null + || $attributeSetIdOfCustomerDestination === null + || $attributeGroupIfOfCustomerSource === null + ) { + return $recordsToSave; + } + + foreach ($recordsToSave as $record) { + $attributeId = $record->getValue('attribute_id'); + $attributeSetId = $record->getValue('attribute_set_id'); + if ($attributeSetId == $attributeSetIdOfCustomerSource) { + $eavEntityAttributeOfCustomerSource[] = $attributeId; + } else if ($attributeSetId == $attributeSetIdOfCustomerDestination) { + $eavEntityAttributeOfCustomerDestination[] = $attributeId; + } + } + $customerAttributeIdsToAdd = array_diff($eavEntityAttributeOfCustomerDestination, $eavEntityAttributeOfCustomerSource); + foreach ($customerAttributeIdsToAdd as $customerAttributeId) { + $dataRecord = [ + 'entity_attribute_id' => null, + 'entity_type_id' => $customerEntityTypeId, + 'attribute_set_id' => $attributeSetIdOfCustomerSource, + 'attribute_group_id' => $attributeGroupIfOfCustomerSource, + 'attribute_id' => $customerAttributeId, + 'sort_order' => 50, + ]; + $destinationRecord = $this->factory->create([ + 'document' => $entityAttributeDocument, + 'data' => $dataRecord + ]); + $recordsToSave->addRecord($destinationRecord); + } + + return $recordsToSave; + } + /** * Migrate tables extended from eav_attribute * From bfee89dcda7382324c65af9c18e6c7a1f856ea84 Mon Sep 17 00:00:00 2001 From: Victor Rad Date: Wed, 29 Mar 2017 14:29:27 +0300 Subject: [PATCH 11/17] MAGETWO-56157: Unable to save Customer's custom Attributes --- etc/ce-to-ce/map-eav.xml.dist | 6 ----- etc/ce-to-ee/map-eav.xml.dist | 6 ----- etc/ee-to-ee/map-eav.xml.dist | 8 ------- src/Migration/Step/Eav/Data.php | 41 +++++++++++++++++++-------------- 4 files changed, 24 insertions(+), 37 deletions(-) diff --git a/etc/ce-to-ce/map-eav.xml.dist b/etc/ce-to-ce/map-eav.xml.dist index 8838998fd..f4ec10005 100644 --- a/etc/ce-to-ce/map-eav.xml.dist +++ b/etc/ce-to-ce/map-eav.xml.dist @@ -151,12 +151,6 @@ - - eav_attribute_group.default_id - - - - catalog_eav_attribute.apply_to diff --git a/etc/ce-to-ee/map-eav.xml.dist b/etc/ce-to-ee/map-eav.xml.dist index 155e44ea8..490506086 100644 --- a/etc/ce-to-ee/map-eav.xml.dist +++ b/etc/ce-to-ee/map-eav.xml.dist @@ -155,12 +155,6 @@ - - eav_attribute_group.default_id - - - - catalog_eav_attribute.is_configurable diff --git a/etc/ee-to-ee/map-eav.xml.dist b/etc/ee-to-ee/map-eav.xml.dist index 94f3e54d3..00daf2d25 100644 --- a/etc/ee-to-ee/map-eav.xml.dist +++ b/etc/ee-to-ee/map-eav.xml.dist @@ -150,20 +150,12 @@ - eav_attribute_group.attribute_group_name - - eav_attribute_group.default_id - - - - - enterprise_rma_item_eav_attribute.input_filter diff --git a/src/Migration/Step/Eav/Data.php b/src/Migration/Step/Eav/Data.php index a72ef93d7..4f5f6090b 100644 --- a/src/Migration/Step/Eav/Data.php +++ b/src/Migration/Step/Eav/Data.php @@ -177,7 +177,7 @@ public function perform() $this->initialData->init(); $this->migrateEntityTypes(); $this->migrateAttributeSetsAndGroups(); - $this->changeOldAttributeSetIdsInEntityTypes(); + $this->changeOldAttributeSetIdsInEntityTypes(['customer', 'customer_address']); $this->migrateAttributes(); $this->migrateAttributesExtended(); $this->migrateEntityAttributes(); @@ -362,9 +362,10 @@ protected function addAttributeGroups($recordsToSave, $documentName, array $grou /** * Change old default attribute set ids in entity types * + * @param array $exceptions * @return void */ - protected function changeOldAttributeSetIdsInEntityTypes() + protected function changeOldAttributeSetIdsInEntityTypes(array $exceptions) { $documentName = 'eav_entity_type'; $destinationDocument = $this->destination->getDocument( @@ -373,7 +374,9 @@ protected function changeOldAttributeSetIdsInEntityTypes() $recordsToSave = $destinationDocument->getRecords(); $entityTypesMigrated = $this->helper->getDestinationRecords($destinationDocument->getName()); foreach ($entityTypesMigrated as $record) { - if (isset($this->defaultAttributeSetIds[$record['entity_type_id']])) { + if (isset($this->defaultAttributeSetIds[$record['entity_type_id']]) + && !in_array($record['entity_type_code'], $exceptions) + ) { $record['default_attribute_set_id'] = $this->defaultAttributeSetIds[$record['entity_type_id']]; } @@ -493,7 +496,7 @@ protected function migrateEntityAttributes() $recordsToSave = $this->processDesignEntityAttributes($recordsToSave); $recordsToSave = $this->moveProductAttributes($recordsToSave); - $recordsToSave = $this->addCustomerAttributes($recordsToSave); + $recordsToSave = $this->addLackingAttributesToCustomerAttributeSet($recordsToSave); $this->destination->clearDocument($destinationDocument->getName()); $this->saveRecords($destinationDocument, $recordsToSave); @@ -675,12 +678,14 @@ private function processDesignEntityAttributes($recordsToSave) } /** - * + * There are attributes from destination customer attribute set + * that do not exit in source customer attribute set. + * The method adds the lacking attributes * * @param Record\Collection $recordsToSave * @return Record\Collection */ - private function addCustomerAttributes($recordsToSave) + private function addLackingAttributesToCustomerAttributeSet($recordsToSave) { $entityAttributeDocument = $this->destination->getDocument( $this->map->getDocumentMap('eav_entity_attribute', MapInterface::TYPE_SOURCE) @@ -697,18 +702,17 @@ private function addCustomerAttributes($recordsToSave) $eavEntityAttributeOfCustomerSource = []; $eavEntityAttributeOfCustomerDestination = []; $attributeGroupIfOfCustomerSource = null; - $recordToAdd = null; foreach ($attributeSets as $attributeSet) { - if ($attributeSet['entity_type_id'] == $customerEntityTypeId - && $attributeSet['attribute_set_name'] == $attributeSetNameCustomerSource - ) { - $attributeSetIdOfCustomerSource = $attributeSet['attribute_set_id']; - } elseif ($attributeSet['entity_type_id'] == $customerEntityTypeId - && $attributeSet['attribute_set_name'] == $attributeSetNameCustomerDestination - ) { - $attributeSetIdOfCustomerDestination = $attributeSet['attribute_set_id']; - } + if ($attributeSet['entity_type_id'] == $customerEntityTypeId + && $attributeSet['attribute_set_name'] == $attributeSetNameCustomerSource + ) { + $attributeSetIdOfCustomerSource = $attributeSet['attribute_set_id']; + } elseif ($attributeSet['entity_type_id'] == $customerEntityTypeId + && $attributeSet['attribute_set_name'] == $attributeSetNameCustomerDestination + ) { + $attributeSetIdOfCustomerDestination = $attributeSet['attribute_set_id']; + } } foreach ($attributeGroups as $attributeGroup) { if ($attributeGroup['attribute_set_id'] == $attributeSetIdOfCustomerSource) { @@ -731,7 +735,10 @@ private function addCustomerAttributes($recordsToSave) $eavEntityAttributeOfCustomerDestination[] = $attributeId; } } - $customerAttributeIdsToAdd = array_diff($eavEntityAttributeOfCustomerDestination, $eavEntityAttributeOfCustomerSource); + $customerAttributeIdsToAdd = array_diff( + $eavEntityAttributeOfCustomerDestination, + $eavEntityAttributeOfCustomerSource + ); foreach ($customerAttributeIdsToAdd as $customerAttributeId) { $dataRecord = [ 'entity_attribute_id' => null, From f3ae348bbe3482aa1790018d46c329531a2d9c35 Mon Sep 17 00:00:00 2001 From: Victor Rad Date: Fri, 31 Mar 2017 11:20:12 +0300 Subject: [PATCH 12/17] MAGETWO-62034: [DB-PPD] Data migration fails on Customer Step --- src/Migration/Step/Customer/Delta.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Migration/Step/Customer/Delta.php b/src/Migration/Step/Customer/Delta.php index 9ec4d3afd..821d26b10 100644 --- a/src/Migration/Step/Customer/Delta.php +++ b/src/Migration/Step/Customer/Delta.php @@ -77,7 +77,7 @@ protected function processChangedRecords($documentName, $idKey) } $destinationName = $this->mapReader->getDocumentMap($documentName, MapInterface::TYPE_SOURCE); - $attributeType = $this->helper->getAttributeType($documentName); + $entityTypeCode = $this->helper->getEntityTypeCodeByDocumentName($documentName); $sourceDocument = $this->source->getDocument($documentName); $destDocument = $this->destination->getDocument($destinationName); @@ -91,7 +91,7 @@ protected function processChangedRecords($documentName, $idKey) echo('.'); $ids[] = $data[$idKey]; - if ($this->helper->isSkipRecord($attributeType, $documentName, $data)) { + if ($this->helper->isSkipRecord($entityTypeCode, $documentName, $data)) { continue; } @@ -103,7 +103,7 @@ protected function processChangedRecords($documentName, $idKey) $destinationRecords ); } - $this->helper->updateAttributeData($attributeType, $documentName, $destinationRecords); + $this->helper->updateAttributeData($entityTypeCode, $documentName, $destinationName, $destinationRecords); $this->destination->updateChangedRecords($destinationName, $destinationRecords); $documentNameDelta = $this->source->getDeltaLogName($documentName); From 370d803f705a716c84d1d14c6553e12e04a29fe9 Mon Sep 17 00:00:00 2001 From: Victor Rad Date: Wed, 5 Apr 2017 12:00:16 +0300 Subject: [PATCH 13/17] MAGETWO-60207: [GitHub] Customer Segment does not work after migration #264 #207 --- etc/ee-to-ee/1.11.0.0/map.xml.dist | 4 +++- etc/ee-to-ee/1.11.0.1/map.xml.dist | 4 +++- etc/ee-to-ee/1.11.0.2/map.xml.dist | 4 +++- etc/ee-to-ee/1.11.1.0/map.xml.dist | 4 +++- etc/ee-to-ee/1.11.2.0/map.xml.dist | 4 +++- etc/ee-to-ee/1.12.0.0/map.xml.dist | 4 +++- etc/ee-to-ee/1.12.0.1/map.xml.dist | 4 +++- etc/ee-to-ee/1.12.0.2/map.xml.dist | 4 +++- etc/ee-to-ee/1.13.0.0/map.xml.dist | 4 +++- etc/ee-to-ee/1.13.0.1/map.xml.dist | 4 +++- etc/ee-to-ee/1.13.0.2/map.xml.dist | 4 +++- etc/ee-to-ee/1.13.1.0/map.xml.dist | 4 +++- etc/ee-to-ee/1.14.0.0/map.xml.dist | 4 +++- etc/ee-to-ee/1.14.0.1/map.xml.dist | 4 +++- etc/ee-to-ee/1.14.1.0/map.xml.dist | 4 +++- etc/ee-to-ee/1.14.2.0/map.xml.dist | 4 +++- etc/ee-to-ee/1.14.2.1/map.xml.dist | 4 +++- etc/ee-to-ee/1.14.2.2/map.xml.dist | 4 +++- etc/ee-to-ee/1.14.2.3/map.xml.dist | 4 +++- etc/ee-to-ee/1.14.2.4/map.xml.dist | 4 +++- etc/ee-to-ee/1.14.3.0/map.xml.dist | 4 +++- etc/ee-to-ee/1.14.3.1/map.xml.dist | 4 +++- etc/ee-to-ee/1.14.3.2/map.xml.dist | 4 +++- 23 files changed, 69 insertions(+), 23 deletions(-) diff --git a/etc/ee-to-ee/1.11.0.0/map.xml.dist b/etc/ee-to-ee/1.11.0.0/map.xml.dist index d35fb17bc..fe3548c04 100644 --- a/etc/ee-to-ee/1.11.0.0/map.xml.dist +++ b/etc/ee-to-ee/1.11.0.0/map.xml.dist @@ -1404,7 +1404,9 @@ enterprise_customersegment_segment.condition_sql - + + + sales_flat_order_item.weee_tax_applied diff --git a/etc/ee-to-ee/1.11.0.1/map.xml.dist b/etc/ee-to-ee/1.11.0.1/map.xml.dist index 48b53e3ee..4715f77e1 100644 --- a/etc/ee-to-ee/1.11.0.1/map.xml.dist +++ b/etc/ee-to-ee/1.11.0.1/map.xml.dist @@ -1404,7 +1404,9 @@ enterprise_customersegment_segment.condition_sql - + + + sales_flat_order_item.weee_tax_applied diff --git a/etc/ee-to-ee/1.11.0.2/map.xml.dist b/etc/ee-to-ee/1.11.0.2/map.xml.dist index 48b53e3ee..4715f77e1 100644 --- a/etc/ee-to-ee/1.11.0.2/map.xml.dist +++ b/etc/ee-to-ee/1.11.0.2/map.xml.dist @@ -1404,7 +1404,9 @@ enterprise_customersegment_segment.condition_sql - + + + sales_flat_order_item.weee_tax_applied diff --git a/etc/ee-to-ee/1.11.1.0/map.xml.dist b/etc/ee-to-ee/1.11.1.0/map.xml.dist index 7350b7f0b..5e7ddd995 100644 --- a/etc/ee-to-ee/1.11.1.0/map.xml.dist +++ b/etc/ee-to-ee/1.11.1.0/map.xml.dist @@ -1404,7 +1404,9 @@ enterprise_customersegment_segment.condition_sql - + + + sales_flat_order_item.weee_tax_applied diff --git a/etc/ee-to-ee/1.11.2.0/map.xml.dist b/etc/ee-to-ee/1.11.2.0/map.xml.dist index 4189db509..906e0a3cb 100644 --- a/etc/ee-to-ee/1.11.2.0/map.xml.dist +++ b/etc/ee-to-ee/1.11.2.0/map.xml.dist @@ -1404,7 +1404,9 @@ enterprise_customersegment_segment.condition_sql - + + + sales_flat_order_item.weee_tax_applied diff --git a/etc/ee-to-ee/1.12.0.0/map.xml.dist b/etc/ee-to-ee/1.12.0.0/map.xml.dist index 6f3288754..554fc883d 100644 --- a/etc/ee-to-ee/1.12.0.0/map.xml.dist +++ b/etc/ee-to-ee/1.12.0.0/map.xml.dist @@ -1374,7 +1374,9 @@ enterprise_customersegment_segment.condition_sql - + + + sales_flat_order_item.weee_tax_applied diff --git a/etc/ee-to-ee/1.12.0.1/map.xml.dist b/etc/ee-to-ee/1.12.0.1/map.xml.dist index 9f1488d90..737cb3462 100644 --- a/etc/ee-to-ee/1.12.0.1/map.xml.dist +++ b/etc/ee-to-ee/1.12.0.1/map.xml.dist @@ -1374,7 +1374,9 @@ enterprise_customersegment_segment.condition_sql - + + + sales_flat_order_item.weee_tax_applied diff --git a/etc/ee-to-ee/1.12.0.2/map.xml.dist b/etc/ee-to-ee/1.12.0.2/map.xml.dist index 9f1488d90..737cb3462 100644 --- a/etc/ee-to-ee/1.12.0.2/map.xml.dist +++ b/etc/ee-to-ee/1.12.0.2/map.xml.dist @@ -1374,7 +1374,9 @@ enterprise_customersegment_segment.condition_sql - + + + sales_flat_order_item.weee_tax_applied diff --git a/etc/ee-to-ee/1.13.0.0/map.xml.dist b/etc/ee-to-ee/1.13.0.0/map.xml.dist index ca720a0e1..3d153be6d 100644 --- a/etc/ee-to-ee/1.13.0.0/map.xml.dist +++ b/etc/ee-to-ee/1.13.0.0/map.xml.dist @@ -1358,7 +1358,9 @@ enterprise_customersegment_segment.condition_sql - + + + sales_flat_order_item.weee_tax_applied diff --git a/etc/ee-to-ee/1.13.0.1/map.xml.dist b/etc/ee-to-ee/1.13.0.1/map.xml.dist index d9ccb0ffd..0437c2447 100644 --- a/etc/ee-to-ee/1.13.0.1/map.xml.dist +++ b/etc/ee-to-ee/1.13.0.1/map.xml.dist @@ -1358,7 +1358,9 @@ enterprise_customersegment_segment.condition_sql - + + + sales_flat_order_item.weee_tax_applied diff --git a/etc/ee-to-ee/1.13.0.2/map.xml.dist b/etc/ee-to-ee/1.13.0.2/map.xml.dist index d9ccb0ffd..0437c2447 100644 --- a/etc/ee-to-ee/1.13.0.2/map.xml.dist +++ b/etc/ee-to-ee/1.13.0.2/map.xml.dist @@ -1358,7 +1358,9 @@ enterprise_customersegment_segment.condition_sql - + + + sales_flat_order_item.weee_tax_applied diff --git a/etc/ee-to-ee/1.13.1.0/map.xml.dist b/etc/ee-to-ee/1.13.1.0/map.xml.dist index 958d6f89a..239105cf6 100644 --- a/etc/ee-to-ee/1.13.1.0/map.xml.dist +++ b/etc/ee-to-ee/1.13.1.0/map.xml.dist @@ -1364,7 +1364,9 @@ enterprise_customersegment_segment.condition_sql - + + + sales_flat_order_item.weee_tax_applied diff --git a/etc/ee-to-ee/1.14.0.0/map.xml.dist b/etc/ee-to-ee/1.14.0.0/map.xml.dist index 0f4d5f964..d413315d6 100644 --- a/etc/ee-to-ee/1.14.0.0/map.xml.dist +++ b/etc/ee-to-ee/1.14.0.0/map.xml.dist @@ -1364,7 +1364,9 @@ enterprise_customersegment_segment.condition_sql - + + + sales_flat_order_item.weee_tax_applied diff --git a/etc/ee-to-ee/1.14.0.1/map.xml.dist b/etc/ee-to-ee/1.14.0.1/map.xml.dist index 0f4d5f964..d413315d6 100644 --- a/etc/ee-to-ee/1.14.0.1/map.xml.dist +++ b/etc/ee-to-ee/1.14.0.1/map.xml.dist @@ -1364,7 +1364,9 @@ enterprise_customersegment_segment.condition_sql - + + + sales_flat_order_item.weee_tax_applied diff --git a/etc/ee-to-ee/1.14.1.0/map.xml.dist b/etc/ee-to-ee/1.14.1.0/map.xml.dist index 386d1ebb0..e7a533774 100644 --- a/etc/ee-to-ee/1.14.1.0/map.xml.dist +++ b/etc/ee-to-ee/1.14.1.0/map.xml.dist @@ -1372,7 +1372,9 @@ enterprise_customersegment_segment.condition_sql - + + + sales_flat_order_item.weee_tax_applied diff --git a/etc/ee-to-ee/1.14.2.0/map.xml.dist b/etc/ee-to-ee/1.14.2.0/map.xml.dist index af8c4aa2a..d60f94064 100644 --- a/etc/ee-to-ee/1.14.2.0/map.xml.dist +++ b/etc/ee-to-ee/1.14.2.0/map.xml.dist @@ -1372,7 +1372,9 @@ enterprise_customersegment_segment.condition_sql - + + + sales_flat_order_item.weee_tax_applied diff --git a/etc/ee-to-ee/1.14.2.1/map.xml.dist b/etc/ee-to-ee/1.14.2.1/map.xml.dist index 2998596de..271240e9a 100644 --- a/etc/ee-to-ee/1.14.2.1/map.xml.dist +++ b/etc/ee-to-ee/1.14.2.1/map.xml.dist @@ -1372,7 +1372,9 @@ enterprise_customersegment_segment.condition_sql - + + + sales_flat_order_item.weee_tax_applied diff --git a/etc/ee-to-ee/1.14.2.2/map.xml.dist b/etc/ee-to-ee/1.14.2.2/map.xml.dist index 4de46fe86..7dcb04713 100644 --- a/etc/ee-to-ee/1.14.2.2/map.xml.dist +++ b/etc/ee-to-ee/1.14.2.2/map.xml.dist @@ -1378,7 +1378,9 @@ enterprise_customersegment_segment.condition_sql - + + + sales_flat_order_item.weee_tax_applied diff --git a/etc/ee-to-ee/1.14.2.3/map.xml.dist b/etc/ee-to-ee/1.14.2.3/map.xml.dist index 59ea20537..5886ea766 100644 --- a/etc/ee-to-ee/1.14.2.3/map.xml.dist +++ b/etc/ee-to-ee/1.14.2.3/map.xml.dist @@ -1378,7 +1378,9 @@ enterprise_customersegment_segment.condition_sql - + + + sales_flat_order_item.weee_tax_applied diff --git a/etc/ee-to-ee/1.14.2.4/map.xml.dist b/etc/ee-to-ee/1.14.2.4/map.xml.dist index 59ea20537..5886ea766 100644 --- a/etc/ee-to-ee/1.14.2.4/map.xml.dist +++ b/etc/ee-to-ee/1.14.2.4/map.xml.dist @@ -1378,7 +1378,9 @@ enterprise_customersegment_segment.condition_sql - + + + sales_flat_order_item.weee_tax_applied diff --git a/etc/ee-to-ee/1.14.3.0/map.xml.dist b/etc/ee-to-ee/1.14.3.0/map.xml.dist index 43e7a887b..0f0f0768c 100644 --- a/etc/ee-to-ee/1.14.3.0/map.xml.dist +++ b/etc/ee-to-ee/1.14.3.0/map.xml.dist @@ -1381,7 +1381,9 @@ enterprise_customersegment_segment.condition_sql - + + + sales_flat_order_item.weee_tax_applied diff --git a/etc/ee-to-ee/1.14.3.1/map.xml.dist b/etc/ee-to-ee/1.14.3.1/map.xml.dist index 43e7a887b..0f0f0768c 100644 --- a/etc/ee-to-ee/1.14.3.1/map.xml.dist +++ b/etc/ee-to-ee/1.14.3.1/map.xml.dist @@ -1381,7 +1381,9 @@ enterprise_customersegment_segment.condition_sql - + + + sales_flat_order_item.weee_tax_applied diff --git a/etc/ee-to-ee/1.14.3.2/map.xml.dist b/etc/ee-to-ee/1.14.3.2/map.xml.dist index 43e7a887b..0f0f0768c 100644 --- a/etc/ee-to-ee/1.14.3.2/map.xml.dist +++ b/etc/ee-to-ee/1.14.3.2/map.xml.dist @@ -1381,7 +1381,9 @@ enterprise_customersegment_segment.condition_sql - + + + sales_flat_order_item.weee_tax_applied From 009c167744698279ef687a97786546155c93d6ab Mon Sep 17 00:00:00 2001 From: Victor Rad Date: Fri, 7 Apr 2017 09:29:20 +0300 Subject: [PATCH 14/17] MAGETWO-64622: Duplicate entry MySQL error during Tier Price step of migration if product has tier and group prices --- src/Migration/Step/TierPrice/Data.php | 2 +- src/Migration/Step/TierPrice/Volume.php | 37 ++++++++++++++++--- .../resource/tier_price/source.sql | 2 + .../Migration/Step/TierPrice/DataTest.php | 2 +- 4 files changed, 35 insertions(+), 8 deletions(-) diff --git a/src/Migration/Step/TierPrice/Data.php b/src/Migration/Step/TierPrice/Data.php index 0879197dc..0d662875d 100644 --- a/src/Migration/Step/TierPrice/Data.php +++ b/src/Migration/Step/TierPrice/Data.php @@ -140,7 +140,7 @@ public function perform() $recordTransformer->transform($sourceRecord, $destRecord); $destinationRecords->addRecord($destRecord); } - $this->destination->saveRecords($destinationName, $destinationRecords); + $this->destination->saveRecords($destinationName, $destinationRecords, true); } $this->progress->finish(LogManager::LOG_LEVEL_DEBUG); } diff --git a/src/Migration/Step/TierPrice/Volume.php b/src/Migration/Step/TierPrice/Volume.php index 094962120..3d7c5bfdb 100644 --- a/src/Migration/Step/TierPrice/Volume.php +++ b/src/Migration/Step/TierPrice/Volume.php @@ -65,13 +65,8 @@ public function perform() $destinationDocuments = $this->helper->getDestinationDocuments(); $this->progress->start(count($sourceDocuments) + count($destinationDocuments)); - $sourceTotal = 0; $destinationTotal = 0; - - foreach ($sourceDocuments as $documentName) { - $sourceTotal += $this->source->getRecordsCount($documentName); - $this->progress->advance(); - } + $sourceTotal = $this->getSourceTotal($sourceDocuments); foreach ($destinationDocuments as $documentName) { $destinationTotal += $this->destination->getRecordsCount($documentName); $this->progress->advance(); @@ -84,4 +79,34 @@ public function perform() $this->progress->finish(); return $this->checkForErrors(Logger::ERROR); } + + /** + * Return number of records with unique key taken into account + * Duplicated records will be omitted + * + * @param array $sourceDocuments + * @return int + */ + private function getSourceTotal(array $sourceDocuments) + { + $sourceRecordsUnique = []; + foreach ($sourceDocuments as $documentName) { + $this->progress->advance(); + $sourceRecords = $this->source->getRecords( + $documentName, + 0, + $this->source->getRecordsCount($documentName) + ); + foreach ($sourceRecords as $record) { + $record['qty'] = isset($record['qty']) ? $record['qty'] : '1.0000'; + $key = $record['entity_id'] . '-' . + $record['all_groups'] . '-' . + $record['customer_group_id'] . '-' . + $record['qty'] . '-' . + $record['website_id']; + $sourceRecordsUnique[$key] = $record; + } + } + return count($sourceRecordsUnique); + } } diff --git a/tests/integration/resource/tier_price/source.sql b/tests/integration/resource/tier_price/source.sql index a8bff7f5e..3460f8a22 100644 --- a/tests/integration/resource/tier_price/source.sql +++ b/tests/integration/resource/tier_price/source.sql @@ -25,6 +25,7 @@ AUTO_INCREMENT=0 ; /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +INSERT INTO `catalog_product_entity_tier_price` (`entity_id`, `all_groups`, `customer_group_id`, `qty`, `value`, `website_id`) VALUES ('1', '0', '1', '1.0000', '15.0000', '0'); INSERT INTO `catalog_product_entity_tier_price` (`entity_id`, `all_groups`, `customer_group_id`, `qty`, `value`, `website_id`) VALUES ('6', '0', '2', '3.0000', '85.0000', '0'); /*!40101 SET SQL_MODE=IFNULL(@OLD_INSERT_SQL_MODE,'') */; @@ -48,5 +49,6 @@ AUTO_INCREMENT=0 ; /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +INSERT INTO `catalog_product_entity_group_price` (`entity_id`, `all_groups`, `customer_group_id`, `value`, `website_id`) VALUES ('1', '0', '1', '10.0000', '0'); INSERT INTO `catalog_product_entity_group_price` (`entity_id`, `all_groups`, `customer_group_id`, `value`, `website_id`) VALUES ('6', '0', '3', '95.0000', '0'); /*!40101 SET SQL_MODE=IFNULL(@OLD_INSERT_SQL_MODE,'') */; diff --git a/tests/integration/testsuite/Migration/Step/TierPrice/DataTest.php b/tests/integration/testsuite/Migration/Step/TierPrice/DataTest.php index d0fb39d91..c996a3cb6 100644 --- a/tests/integration/testsuite/Migration/Step/TierPrice/DataTest.php +++ b/tests/integration/testsuite/Migration/Step/TierPrice/DataTest.php @@ -88,6 +88,6 @@ public function testPerform() $this->mapFactory ); $this->assertTrue($data->perform()); - $this->assertEquals(2, count($this->destination->getRecords(self::TIRE_PRICE_TABLE_DESTINATION, 0))); + $this->assertEquals(3, count($this->destination->getRecords(self::TIRE_PRICE_TABLE_DESTINATION, 0))); } } From a05aac1575abc4dd12c6906bf0e9d1a393228973 Mon Sep 17 00:00:00 2001 From: Victor Rad Date: Wed, 12 Apr 2017 10:17:37 +0300 Subject: [PATCH 15/17] MAGETWO-67347: Make Bamboo tests green --- src/Migration/Step/Eav/Data.php | 2 ++ tests/integration/resource/eav/source.sql | 1 + tests/integration/resource/ee.eav/source.sql | 1 + .../Migration/Step/Customer/HelperTest.php | 30 +++++++++++++++---- .../Eav/Integrity/AttributeGroupNamesTest.php | 5 +++- 5 files changed, 33 insertions(+), 6 deletions(-) diff --git a/src/Migration/Step/Eav/Data.php b/src/Migration/Step/Eav/Data.php index 4f5f6090b..a083e2940 100644 --- a/src/Migration/Step/Eav/Data.php +++ b/src/Migration/Step/Eav/Data.php @@ -23,6 +23,8 @@ * @SuppressWarnings(PHPMD.CyclomaticComplexity) * @SuppressWarnings(PHPMD.CouplingBetweenObjects) * @SuppressWarnings(PHPMD.TooManyFields) + * @SuppressWarnings(PHPMD.ExcessiveClassComplexity) + * @SuppressWarnings(PHPMD.NPathComplexity) * @codeCoverageIgnoreStart */ class Data implements StageInterface, RollbackInterface diff --git a/tests/integration/resource/eav/source.sql b/tests/integration/resource/eav/source.sql index 6db27baa6..ffa3ab7fb 100644 --- a/tests/integration/resource/eav/source.sql +++ b/tests/integration/resource/eav/source.sql @@ -129,6 +129,7 @@ INSERT INTO `eav_attribute_group` (`attribute_group_id`, `attribute_set_id`, `at INSERT INTO `eav_attribute_group` (`attribute_group_id`, `attribute_set_id`, `attribute_group_name`, `sort_order`, `default_id`) VALUES (1, 1, 'General', 1, 1); INSERT INTO `eav_attribute_group` (`attribute_group_id`, `attribute_set_id`, `attribute_group_name`, `sort_order`, `default_id`) VALUES (5, 4, 'Prices', 2, 0); INSERT INTO `eav_attribute_group` (`attribute_group_id`, `attribute_set_id`, `attribute_group_name`, `sort_order`, `default_id`) VALUES (6, 4, 'Design', 3, 0); +INSERT INTO `eav_attribute_group` (`attribute_group_id`, `attribute_set_id`, `attribute_group_name`, `sort_order`, `default_id`) VALUES (7, 4, 'Images', 4, 0); CREATE TABLE `eav_entity_attribute` ( `entity_attribute_id` INT(10) UNSIGNED NOT NULL COMMENT 'Entity Attribute Id', diff --git a/tests/integration/resource/ee.eav/source.sql b/tests/integration/resource/ee.eav/source.sql index 538f34f32..3d969590c 100644 --- a/tests/integration/resource/ee.eav/source.sql +++ b/tests/integration/resource/ee.eav/source.sql @@ -129,6 +129,7 @@ INSERT INTO `eav_attribute_group` (`attribute_group_id`, `attribute_set_id`, `at INSERT INTO `eav_attribute_group` (`attribute_group_id`, `attribute_set_id`, `attribute_group_name`, `sort_order`, `default_id`) VALUES (1, 1, 'General', 1, 1); INSERT INTO `eav_attribute_group` (`attribute_group_id`, `attribute_set_id`, `attribute_group_name`, `sort_order`, `default_id`) VALUES (5, 4, 'Prices', 2, 0); INSERT INTO `eav_attribute_group` (`attribute_group_id`, `attribute_set_id`, `attribute_group_name`, `sort_order`, `default_id`) VALUES (6, 4, 'Design', 3, 0); +INSERT INTO `eav_attribute_group` (`attribute_group_id`, `attribute_set_id`, `attribute_group_name`, `sort_order`, `default_id`) VALUES (7, 4, 'Images', 4, 0); CREATE TABLE `eav_entity_attribute` ( `entity_attribute_id` INT(10) UNSIGNED NOT NULL COMMENT 'Entity Attribute Id', diff --git a/tests/unit/testsuite/Migration/Step/Customer/HelperTest.php b/tests/unit/testsuite/Migration/Step/Customer/HelperTest.php index 14720f61f..534bf96f7 100644 --- a/tests/unit/testsuite/Migration/Step/Customer/HelperTest.php +++ b/tests/unit/testsuite/Migration/Step/Customer/HelperTest.php @@ -10,6 +10,10 @@ use Migration\ResourceModel\Record\Collection; use Migration\Step\Customer\Helper; +/** + * Class HelperTest + * @SuppressWarnings(PHPMD) + */ class HelperTest extends \PHPUnit_Framework_TestCase { /** @@ -176,8 +180,12 @@ public function testUpdateAttributeData( ->method('getOption') ->willReturnMap([$upgradePasswordHash]); - $testMethodArguments[] = $destinationRecords; - $this->helper->updateAttributeData(...$testMethodArguments); + $this->helper->updateAttributeData( + $testMethodArguments['entityTypeCode'], + $testMethodArguments['sourceDocName'], + $testMethodArguments['destinationDocName'], + $destinationRecords + ); foreach ($destinationRecords as $key => $record) { $this->assertEquals($record->getData(), $recordsResult[$key]); @@ -261,7 +269,11 @@ public function fixturesDataProvider() { return [ [ - 'testMethodArguments' => ['customer', 'customer_entity', 'customer_entity'], + 'testMethodArguments' => [ + 'entityTypeCode' => 'customer', + 'sourceDocName' => 'customer_entity', + 'destinationDocName' => 'customer_entity' + ], 'recordsData' => [ ['entity_id' => '1', 'entity_type_id' => '1', 'email' => 'customer1@example.com'], ['entity_id' => '2', 'entity_type_id' => '1', 'email' => 'customer2@example.com'], @@ -312,7 +324,11 @@ public function fixturesDataProvider() ] ], ], [ - 'testMethodArguments' => ['customer', 'customer_entity', 'customer_entity'], + 'testMethodArguments' => [ + 'entityTypeCode' => 'customer', + 'sourceDocName' => 'customer_entity', + 'destinationDocName' => 'customer_entity' + ], 'recordsData' => [ ['entity_id' => '1', 'entity_type_id' => '1', 'email' => 'customer1@example.com'], ['entity_id' => '2', 'entity_type_id' => '1', 'email' => 'customer2@example.com'] @@ -345,7 +361,11 @@ public function fixturesDataProvider() ] ], ], [ - 'testMethodArguments' => ['customer_address', 'customer_address_entity', 'customer_address_entity'], + 'testMethodArguments' => [ + 'entityTypeCode' => 'customer_address', + 'sourceDocName' => 'customer_address_entity', + 'destinationDocName' => 'customer_address_entity' + ], 'recordsData' => [ ['entity_id' => '1', 'city' => 'Austin'], ['entity_id' => '2', 'city' => 'Kiev'] diff --git a/tests/unit/testsuite/Migration/Step/Eav/Integrity/AttributeGroupNamesTest.php b/tests/unit/testsuite/Migration/Step/Eav/Integrity/AttributeGroupNamesTest.php index 41547eefe..07b77ce63 100644 --- a/tests/unit/testsuite/Migration/Step/Eav/Integrity/AttributeGroupNamesTest.php +++ b/tests/unit/testsuite/Migration/Step/Eav/Integrity/AttributeGroupNamesTest.php @@ -77,7 +77,8 @@ public function getFixtureData() 'eav_attribute_group' => [ 7 => ['attribute_set_id' => 4, 'attribute_group_name' => 'General'], 8 => ['attribute_set_id' => 4, 'attribute_group_name' => 'Prices'], - 11 => ['attribute_set_id' => 4, 'attribute_group_name' => 'Design'] + 11 => ['attribute_set_id' => 4, 'attribute_group_name' => 'Design'], + 13 => ['attribute_set_id' => 4, 'attribute_group_name' => 'Images'] ], 'eav_entity_type' => [ 'catalog_product' => ['entity_type_id' => 4] @@ -95,6 +96,7 @@ public function getFixtureData() 7 => ['attribute_set_id' => 3, 'attribute_group_name' => 'General'], 8 => ['attribute_set_id' => 3, 'attribute_group_name' => 'Prices'], 11 => ['attribute_set_id' => 3, 'attribute_group_name' => 'Design'], + 13 => ['attribute_set_id' => 3, 'attribute_group_name' => 'Images'], 12 => ['attribute_set_id' => 4, 'attribute_group_name' => 'General'] ], 'eav_entity_type' => [ @@ -115,6 +117,7 @@ public function getFixtureData() 7 => ['attribute_set_id' => 3, 'attribute_group_name' => 'General'], 8 => ['attribute_set_id' => 3, 'attribute_group_name' => 'Prices'], 11 => ['attribute_set_id' => 3, 'attribute_group_name' => 'Design'], + 13 => ['attribute_set_id' => 3, 'attribute_group_name' => 'Images'], 12 => ['attribute_set_id' => 4, 'attribute_group_name' => 'General'] ], 'eav_entity_type' => [ From 6228422e17c79a85ab74df751964f09ce6d68f3b Mon Sep 17 00:00:00 2001 From: Victor Rad Date: Wed, 12 Apr 2017 15:23:57 +0300 Subject: [PATCH 16/17] MAGETWO-67348: DMT 2.1.6 Publication --- CHANGELOG.md | 14 ++++++++++++++ composer.json | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d39178e5..98b9ce6db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,17 @@ +2.1.6 +============= +* Added possibility to specify custom port for connection to MySQL server +* Added support for versions: + + * Magento CE: 2.1.6 + * Magento EE: 2.1.6 + +* Fixed bugs: + * Unable to save Customer's custom Attribute value if the attribute was created after migration + * An error occurred in the TierPrice step if there were Group Price and Tier Price with qty 1 + * Issues [#207](https://github.com/magento/data-migration-tool/issues/207), [#264](https://github.com/magento/data-migration-tool/issues/264): Customer Segments with product attributes in conditions caused Store Front to be inoperative + * [Issue #212](https://github.com/magento/data-migration-tool/issues/212): In case system attributes category_ids, giftcard_type, price_type, shipment_type, sku_type, weight_type, swatch_image are not part of product attribute set it will be checked and linked to it + 2.1.5 ============= * Updated copyright notice diff --git a/composer.json b/composer.json index cf169cd8f..2ce94a74b 100644 --- a/composer.json +++ b/composer.json @@ -1,7 +1,7 @@ { "name": "magento/data-migration-tool", "description": "Migration Tool", - "version": "2.1.5", + "version": "2.1.6", "require": { "symfony/console": "~2.3", "magento/framework": "~100.1", From 2f6b723d4fe09f3cebfafbcd0d9a7de6eca895c4 Mon Sep 17 00:00:00 2001 From: Victor Rad Date: Thu, 13 Apr 2017 17:39:53 +0300 Subject: [PATCH 17/17] MAGETWO-67348: DMT 2.1.6 Publication --- CHANGELOG.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 98b9ce6db..0cbcdda7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,16 +1,16 @@ 2.1.6 ============= -* Added possibility to specify custom port for connection to MySQL server +* Added possibility to specify custom port for MySQL server connection * Added support for versions: * Magento CE: 2.1.6 * Magento EE: 2.1.6 * Fixed bugs: - * Unable to save Customer's custom Attribute value if the attribute was created after migration - * An error occurred in the TierPrice step if there were Group Price and Tier Price with qty 1 + * Unable to save Customer's custom attribute value if the attribute has been created after migration + * An error occurred during the TierPrice migration step if the product had both Tier and Group Prices and the quantity of a product in Tier Price was 1 * Issues [#207](https://github.com/magento/data-migration-tool/issues/207), [#264](https://github.com/magento/data-migration-tool/issues/264): Customer Segments with product attributes in conditions caused Store Front to be inoperative - * [Issue #212](https://github.com/magento/data-migration-tool/issues/212): In case system attributes category_ids, giftcard_type, price_type, shipment_type, sku_type, weight_type, swatch_image are not part of product attribute set it will be checked and linked to it + * [Issue #212](https://github.com/magento/data-migration-tool/issues/212): When some system attributes had not been included in a product attribute set, they were checked and linked to such set instead of being added automatically. These system attributes are: category_ids, giftcard_type, price_type, shipment_type, sku_type, weight_type, swatch_image 2.1.5 =============